<!DOCTYPE html>
<!-- saved from url=(0042)https://zybuluo.com/hanbingtao/note/626300 -->
<html class="theme theme-white"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
    
    
    <meta name="description" content="Cmd Markdown 编辑阅读器，支持实时同步预览，区分写作和阅读模式，支持在线存储，分享文稿网址。">
    <meta name="author" content="Jiawei Zhang">

    <meta name="viewport" content="width=device-width, initial-scale=1.0">

    
    <title>零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器</title>


    <link href="https://zybuluo.com/static/img/favicon.png" type="image/x-icon" rel="icon">

    <link href="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/1bc053c8.base.lib.min.css" rel="stylesheet" media="screen">


    
    <!-- id="prettify-style" will be used to get the link element below and change href to change prettify code, so it can't be in beginmin/endmin block. -->
    <link id="prettify-style" href="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/prettify-cmd.css" type="text/css" rel="stylesheet">
    <!--
    <link id="mermaid-style" href="https://zybuluo.com/static/editor/libs/mermaid/mermaid.forest.css" type="text/css" rel="stylesheet">
    -->
    <link href="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/45c7d56d.layout.min.css" rel="stylesheet" media="screen">


    

    <script async="" src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/analytics.js"></script><script>
      (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
      (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
      m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
      })(window,document,'script','//www.google-analytics.com/analytics.js','ga');

      ga('create', 'UA-44461741-1', 'zybuluo.com');
      ga('send', 'pageview');
    </script>
<style type="text/css">.MathJax_Hover_Frame {border-radius: .25em; -webkit-border-radius: .25em; -moz-border-radius: .25em; -khtml-border-radius: .25em; box-shadow: 0px 0px 15px #83A; -webkit-box-shadow: 0px 0px 15px #83A; -moz-box-shadow: 0px 0px 15px #83A; -khtml-box-shadow: 0px 0px 15px #83A; border: 1px solid #A6D ! important; display: inline-block; position: absolute}
.MathJax_Hover_Arrow {position: absolute; width: 15px; height: 11px; cursor: pointer}
</style><style type="text/css">#MathJax_About {position: fixed; left: 50%; width: auto; text-align: center; border: 3px outset; padding: 1em 2em; background-color: #DDDDDD; color: black; cursor: default; font-family: message-box; font-size: 120%; font-style: normal; text-indent: 0; text-transform: none; line-height: normal; letter-spacing: normal; word-spacing: normal; word-wrap: normal; white-space: nowrap; float: none; z-index: 201; border-radius: 15px; -webkit-border-radius: 15px; -moz-border-radius: 15px; -khtml-border-radius: 15px; box-shadow: 0px 10px 20px #808080; -webkit-box-shadow: 0px 10px 20px #808080; -moz-box-shadow: 0px 10px 20px #808080; -khtml-box-shadow: 0px 10px 20px #808080; filter: progid:DXImageTransform.Microsoft.dropshadow(OffX=2, OffY=2, Color='gray', Positive='true')}
.MathJax_Menu {position: absolute; background-color: white; color: black; width: auto; padding: 5px 0px; border: 1px solid #CCCCCC; margin: 0; cursor: default; font: menu; text-align: left; text-indent: 0; text-transform: none; line-height: normal; letter-spacing: normal; word-spacing: normal; word-wrap: normal; white-space: nowrap; float: none; z-index: 201; border-radius: 5px; -webkit-border-radius: 5px; -moz-border-radius: 5px; -khtml-border-radius: 5px; box-shadow: 0px 10px 20px #808080; -webkit-box-shadow: 0px 10px 20px #808080; -moz-box-shadow: 0px 10px 20px #808080; -khtml-box-shadow: 0px 10px 20px #808080; filter: progid:DXImageTransform.Microsoft.dropshadow(OffX=2, OffY=2, Color='gray', Positive='true')}
.MathJax_MenuItem {padding: 1px 2em; background: transparent}
.MathJax_MenuArrow {position: absolute; right: .5em; color: #666666}
.MathJax_MenuActive .MathJax_MenuArrow {color: white}
.MathJax_MenuCheck {position: absolute; left: .7em}
.MathJax_MenuRadioCheck {position: absolute; left: .7em}
.MathJax_MenuLabel {padding: 1px 2em 3px 1.33em; font-style: italic}
.MathJax_MenuRule {border-top: 1px solid #DDDDDD; margin: 4px 3px}
.MathJax_MenuDisabled {color: GrayText}
.MathJax_MenuActive {background-color: #606872; color: white}
.MathJax_Menu_Close {position: absolute; width: 31px; height: 31px; top: -15px; left: -15px}
</style><style type="text/css">#MathJax_Zoom {position: absolute; background-color: #F0F0F0; overflow: auto; display: block; z-index: 301; padding: .5em; border: 1px solid black; margin: 0; font-weight: normal; font-style: normal; text-align: left; text-indent: 0; text-transform: none; line-height: normal; letter-spacing: normal; word-spacing: normal; word-wrap: normal; white-space: nowrap; float: none; box-shadow: 5px 5px 15px #AAAAAA; -webkit-box-shadow: 5px 5px 15px #AAAAAA; -moz-box-shadow: 5px 5px 15px #AAAAAA; -khtml-box-shadow: 5px 5px 15px #AAAAAA; filter: progid:DXImageTransform.Microsoft.dropshadow(OffX=2, OffY=2, Color='gray', Positive='true')}
#MathJax_ZoomOverlay {position: absolute; left: 0; top: 0; z-index: 300; display: inline-block; width: 100%; height: 100%; border: 0; padding: 0; margin: 0; background-color: white; opacity: 0; filter: alpha(opacity=0)}
#MathJax_ZoomFrame {position: relative; display: inline-block; height: 0; width: 0}
#MathJax_ZoomEventTrap {position: absolute; left: 0; top: 0; z-index: 302; display: inline-block; border: 0; padding: 0; margin: 0; background-color: white; opacity: 0; filter: alpha(opacity=0)}
</style><style type="text/css">.MathJax_Preview {color: #888}
#MathJax_Message {position: fixed; left: 1em; bottom: 1.5em; background-color: #E6E6E6; border: 1px solid #959595; margin: 0px; padding: 2px 8px; z-index: 102; color: black; font-size: 80%; width: auto; white-space: nowrap}
#MathJax_MSIE_Frame {position: absolute; top: 0; left: 0; width: 0px; z-index: 101; border: 0px; margin: 0px; padding: 0px}
.MathJax_Error {color: #CC0000; font-style: italic}
</style><style type="text/css">.MathJax_SVG_Display {text-align: center; margin: 1em 0em; position: relative; display: block; width: 100%}
#MathJax_SVG_Tooltip {background-color: InfoBackground; color: InfoText; border: 1px solid black; box-shadow: 2px 2px 5px #AAAAAA; -webkit-box-shadow: 2px 2px 5px #AAAAAA; -moz-box-shadow: 2px 2px 5px #AAAAAA; -khtml-box-shadow: 2px 2px 5px #AAAAAA; padding: 3px 4px; z-index: 401; position: absolute; left: 0; top: 0; width: auto; height: auto; display: none}
.MathJax_SVG {display: inline; font-style: normal; font-weight: normal; line-height: normal; font-size: 100%; font-size-adjust: none; text-indent: 0; text-align: left; text-transform: none; letter-spacing: normal; word-spacing: normal; word-wrap: normal; white-space: nowrap; float: none; direction: ltr; border: 0; padding: 0; margin: 0}
.MathJax_SVG * {transition: none; -webkit-transition: none; -moz-transition: none; -ms-transition: none; -o-transition: none}
.mjx-svg-href {fill: blue; stroke: blue}
.MathJax_SVG_Processing {visibility: hidden; position: absolute; top: 0; left: 0; width: 0; height: 0; overflow: hidden; display: block}
.MathJax_SVG_Processed {display: none!important}
.MathJax_SVG_ExBox {display: block; overflow: hidden; width: 1px; height: 60ex}
.MathJax_SVG .noError {vertical-align: ; font-size: 90%; text-align: left; color: black; padding: 1px 3px; border: 1px solid}
</style></head>

<body class="theme  theme-white pace-done"><div style="visibility: hidden; overflow: hidden; position: absolute; top: 0px; height: 1px; width: auto; padding: 0px; border: 0px; margin: 0px; text-align: left; text-indent: 0px; text-transform: none; line-height: normal; letter-spacing: normal; word-spacing: normal;"><div id="MathJax_SVG_Hidden"></div><svg><defs id="MathJax_SVG_glyphs"><path id="MJMAINB-63" stroke-width="1" d="M447 131H458Q478 131 478 117Q478 112 471 95T439 51T377 9Q330 -6 286 -6Q196 -6 135 35Q39 96 39 222Q39 324 101 384Q169 453 286 453Q359 453 411 431T464 353Q464 319 445 302T395 284Q360 284 343 305T325 353Q325 380 338 396H333Q317 398 295 398H292Q280 398 271 397T245 390T218 373T197 338T183 283Q182 275 182 231Q182 199 184 180T193 132T220 85T270 57Q289 50 317 50H326Q385 50 414 115Q419 127 423 129T447 131Z"></path><path id="MJMAIN-31" stroke-width="1" d="M213 578L200 573Q186 568 160 563T102 556H83V602H102Q149 604 189 617T245 641T273 663Q275 666 285 666Q294 666 302 660V361L303 61Q310 54 315 52T339 48T401 46H427V0H416Q395 3 257 3Q121 3 100 0H88V46H114Q136 46 152 46T177 47T193 50T201 52T207 57T213 61V578Z"></path><path id="MJMAIN-32" stroke-width="1" d="M109 429Q82 429 66 447T50 491Q50 562 103 614T235 666Q326 666 387 610T449 465Q449 422 429 383T381 315T301 241Q265 210 201 149L142 93L218 92Q375 92 385 97Q392 99 409 186V189H449V186Q448 183 436 95T421 3V0H50V19V31Q50 38 56 46T86 81Q115 113 136 137Q145 147 170 174T204 211T233 244T261 278T284 308T305 340T320 369T333 401T340 431T343 464Q343 527 309 573T212 619Q179 619 154 602T119 569T109 550Q109 549 114 549Q132 549 151 535T170 489Q170 464 154 447T109 429Z"></path><path id="MJMAINB-70" stroke-width="1" d="M32 442L123 446Q214 450 215 450H221V409Q222 409 229 413T251 423T284 436T328 446T382 450Q480 450 540 388T600 223Q600 128 539 61T361 -6H354Q292 -6 236 28L227 34V-132H296V-194H287Q269 -191 163 -191Q56 -191 38 -194H29V-132H98V113V284Q98 330 97 348T93 370T83 376Q69 380 42 380H29V442H32ZM457 224Q457 303 427 349T350 395Q282 395 235 352L227 345V104L233 97Q274 45 337 45Q383 45 420 86T457 224Z"></path><path id="MJMATHI-57" stroke-width="1" d="M436 683Q450 683 486 682T553 680Q604 680 638 681T677 682Q695 682 695 674Q695 670 692 659Q687 641 683 639T661 637Q636 636 621 632T600 624T597 615Q597 603 613 377T629 138L631 141Q633 144 637 151T649 170T666 200T690 241T720 295T759 362Q863 546 877 572T892 604Q892 619 873 628T831 637Q817 637 817 647Q817 650 819 660Q823 676 825 679T839 682Q842 682 856 682T895 682T949 681Q1015 681 1034 683Q1048 683 1048 672Q1048 666 1045 655T1038 640T1028 637Q1006 637 988 631T958 617T939 600T927 584L923 578L754 282Q586 -14 585 -15Q579 -22 561 -22Q546 -22 542 -17Q539 -14 523 229T506 480L494 462Q472 425 366 239Q222 -13 220 -15T215 -19Q210 -22 197 -22Q178 -22 176 -15Q176 -12 154 304T131 622Q129 631 121 633T82 637H58Q51 644 51 648Q52 671 64 683H76Q118 680 176 680Q301 680 313 683H323Q329 677 329 674T327 656Q322 641 318 637H297Q236 634 232 620Q262 160 266 136L501 550L499 587Q496 629 489 632Q483 636 447 637Q428 637 422 639T416 648Q416 650 418 660Q419 664 420 669T421 676T424 680T428 682T436 683Z"></path><path id="MJMATHI-64" stroke-width="1" d="M366 683Q367 683 438 688T511 694Q523 694 523 686Q523 679 450 384T375 83T374 68Q374 26 402 26Q411 27 422 35Q443 55 463 131Q469 151 473 152Q475 153 483 153H487H491Q506 153 506 145Q506 140 503 129Q490 79 473 48T445 8T417 -8Q409 -10 393 -10Q359 -10 336 5T306 36L300 51Q299 52 296 50Q294 48 292 46Q233 -10 172 -10Q117 -10 75 30T33 157Q33 205 53 255T101 341Q148 398 195 420T280 442Q336 442 364 400Q369 394 369 396Q370 400 396 505T424 616Q424 629 417 632T378 637H357Q351 643 351 645T353 664Q358 683 366 683ZM352 326Q329 405 277 405Q242 405 210 374T160 293Q131 214 119 129Q119 126 119 118T118 106Q118 61 136 44T179 26Q233 26 290 98L298 109L352 326Z"></path><path id="MJMAIN-D7" stroke-width="1" d="M630 29Q630 9 609 9Q604 9 587 25T493 118L389 222L284 117Q178 13 175 11Q171 9 168 9Q160 9 154 15T147 29Q147 36 161 51T255 146L359 250L255 354Q174 435 161 449T147 471Q147 480 153 485T168 490Q173 490 175 489Q178 487 284 383L389 278L493 382Q570 459 587 475T609 491Q630 491 630 471Q630 464 620 453T522 355L418 250L522 145Q606 61 618 48T630 29Z"></path><path id="MJMAIN-3D" stroke-width="1" d="M56 347Q56 360 70 367H707Q722 359 722 347Q722 336 708 328L390 327H72Q56 332 56 347ZM56 153Q56 168 72 173H708Q722 163 722 153Q722 140 707 133H70Q56 140 56 153Z"></path><path id="MJMATHI-74" stroke-width="1" d="M26 385Q19 392 19 395Q19 399 22 411T27 425Q29 430 36 430T87 431H140L159 511Q162 522 166 540T173 566T179 586T187 603T197 615T211 624T229 626Q247 625 254 615T261 596Q261 589 252 549T232 470L222 433Q222 431 272 431H323Q330 424 330 420Q330 398 317 385H210L174 240Q135 80 135 68Q135 26 162 26Q197 26 230 60T283 144Q285 150 288 151T303 153H307Q322 153 322 145Q322 142 319 133Q314 117 301 95T267 48T216 6T155 -11Q125 -11 98 4T59 56Q57 64 57 83V101L92 241Q127 382 128 383Q128 385 77 385H26Z"></path><path id="MJMATHI-61" stroke-width="1" d="M33 157Q33 258 109 349T280 441Q331 441 370 392Q386 422 416 422Q429 422 439 414T449 394Q449 381 412 234T374 68Q374 43 381 35T402 26Q411 27 422 35Q443 55 463 131Q469 151 473 152Q475 153 483 153H487Q506 153 506 144Q506 138 501 117T481 63T449 13Q436 0 417 -8Q409 -10 393 -10Q359 -10 336 5T306 36L300 51Q299 52 296 50Q294 48 292 46Q233 -10 172 -10Q117 -10 75 30T33 157ZM351 328Q351 334 346 350T323 385T277 405Q242 405 210 374T160 293Q131 214 119 129Q119 126 119 118T118 106Q118 61 136 44T179 26Q217 26 254 59T298 110Q300 114 325 217T351 328Z"></path><path id="MJMATHI-6E" stroke-width="1" d="M21 287Q22 293 24 303T36 341T56 388T89 425T135 442Q171 442 195 424T225 390T231 369Q231 367 232 367L243 378Q304 442 382 442Q436 442 469 415T503 336T465 179T427 52Q427 26 444 26Q450 26 453 27Q482 32 505 65T540 145Q542 153 560 153Q580 153 580 145Q580 144 576 130Q568 101 554 73T508 17T439 -10Q392 -10 371 17T350 73Q350 92 386 193T423 345Q423 404 379 404H374Q288 404 229 303L222 291L189 157Q156 26 151 16Q138 -11 108 -11Q95 -11 87 -5T76 7T74 17Q74 30 112 180T152 343Q153 348 153 366Q153 405 129 405Q91 405 66 305Q60 285 60 284Q58 278 41 278H27Q21 284 21 287Z"></path><path id="MJMATHI-68" stroke-width="1" d="M137 683Q138 683 209 688T282 694Q294 694 294 685Q294 674 258 534Q220 386 220 383Q220 381 227 388Q288 442 357 442Q411 442 444 415T478 336Q478 285 440 178T402 50Q403 36 407 31T422 26Q450 26 474 56T513 138Q516 149 519 151T535 153Q555 153 555 145Q555 144 551 130Q535 71 500 33Q466 -10 419 -10H414Q367 -10 346 17T325 74Q325 90 361 192T398 345Q398 404 354 404H349Q266 404 205 306L198 293L164 158Q132 28 127 16Q114 -11 83 -11Q69 -11 59 -2T48 16Q48 30 121 320L195 616Q195 629 188 632T149 637H128Q122 643 122 645T124 664Q129 683 137 683Z"></path><path id="MJMAIN-28" stroke-width="1" d="M94 250Q94 319 104 381T127 488T164 576T202 643T244 695T277 729T302 750H315H319Q333 750 333 741Q333 738 316 720T275 667T226 581T184 443T167 250T184 58T225 -81T274 -167T316 -220T333 -241Q333 -250 318 -250H315H302L274 -226Q180 -141 137 -14T94 250Z"></path><path id="MJMAIN-5B" stroke-width="1" d="M118 -250V750H255V710H158V-210H255V-250H118Z"></path><path id="MJMAIN-5D" stroke-width="1" d="M22 710V750H159V-250H22V-210H119V710H22Z"></path><path id="MJSZ3-5B" stroke-width="1" d="M247 -949V1450H516V1388H309V-887H516V-949H247Z"></path><path id="MJSZ3-5D" stroke-width="1" d="M11 1388V1450H280V-949H11V-887H218V1388H11Z"></path><path id="MJMAIN-2B" stroke-width="1" d="M56 237T56 250T70 270H369V420L370 570Q380 583 389 583Q402 583 409 568V270H707Q722 262 722 250T707 230H409V-68Q401 -82 391 -82H389H387Q375 -82 369 -68V230H70Q56 237 56 250Z"></path><path id="MJMAINB-62" stroke-width="1" d="M32 686L123 690Q214 694 215 694H221V409Q289 450 378 450Q479 450 539 387T600 221Q600 122 535 58T358 -6H355Q272 -6 203 53L160 1L129 0H98V301Q98 362 98 435T99 525Q99 591 97 604T83 620Q69 624 42 624H29V686H32ZM227 105L232 99Q237 93 242 87T258 73T280 59T306 49T339 45Q380 45 411 66T451 131Q457 160 457 230Q457 264 456 284T448 329T430 367T396 389T343 398Q282 398 235 355L227 348V105Z"></path><path id="MJMAIN-29" stroke-width="1" d="M60 749L64 750Q69 750 74 750H86L114 726Q208 641 251 514T294 250Q294 182 284 119T261 12T224 -76T186 -143T145 -194T113 -227T90 -246Q87 -249 86 -250H74Q66 -250 63 -250T58 -247T55 -238Q56 -237 66 -225Q221 -64 221 250T66 725Q56 737 55 738Q55 746 60 749Z"></path><path id="MJMAIN-33" stroke-width="1" d="M127 463Q100 463 85 480T69 524Q69 579 117 622T233 665Q268 665 277 664Q351 652 390 611T430 522Q430 470 396 421T302 350L299 348Q299 347 308 345T337 336T375 315Q457 262 457 175Q457 96 395 37T238 -22Q158 -22 100 21T42 130Q42 158 60 175T105 193Q133 193 151 175T169 130Q169 119 166 110T159 94T148 82T136 74T126 70T118 67L114 66Q165 21 238 21Q293 21 321 74Q338 107 338 175V195Q338 290 274 322Q259 328 213 329L171 330L168 332Q166 335 166 348Q166 366 174 366Q202 366 232 371Q266 376 294 413T322 525V533Q322 590 287 612Q265 626 240 626Q208 626 181 615T143 592T132 580H135Q138 579 143 578T153 573T165 566T175 555T183 540T186 520Q186 498 172 481T127 463Z"></path><path id="MJMATHI-3B4" stroke-width="1" d="M195 609Q195 656 227 686T302 717Q319 716 351 709T407 697T433 690Q451 682 451 662Q451 644 438 628T403 612Q382 612 348 641T288 671T249 657T235 628Q235 584 334 463Q401 379 401 292Q401 169 340 80T205 -10H198Q127 -10 83 36T36 153Q36 286 151 382Q191 413 252 434Q252 435 245 449T230 481T214 521T201 566T195 609ZM112 130Q112 83 136 55T204 27Q233 27 256 51T291 111T309 178T316 232Q316 267 309 298T295 344T269 400L259 396Q215 381 183 342T137 256T118 179T112 130Z"></path><path id="MJMATHI-6B" stroke-width="1" d="M121 647Q121 657 125 670T137 683Q138 683 209 688T282 694Q294 694 294 686Q294 679 244 477Q194 279 194 272Q213 282 223 291Q247 309 292 354T362 415Q402 442 438 442Q468 442 485 423T503 369Q503 344 496 327T477 302T456 291T438 288Q418 288 406 299T394 328Q394 353 410 369T442 390L458 393Q446 405 434 405H430Q398 402 367 380T294 316T228 255Q230 254 243 252T267 246T293 238T320 224T342 206T359 180T365 147Q365 130 360 106T354 66Q354 26 381 26Q429 26 459 145Q461 153 479 153H483Q499 153 499 144Q499 139 496 130Q455 -11 378 -11Q333 -11 305 15T277 90Q277 108 280 121T283 145Q283 167 269 183T234 206T200 217T182 220H180Q168 178 159 139T145 81T136 44T129 20T122 7T111 -2Q98 -11 83 -11Q66 -11 57 -1T48 16Q48 26 85 176T158 471L195 616Q196 629 188 632T149 637H144Q134 637 131 637T124 640T121 647Z"></path><path id="MJMATHI-70" stroke-width="1" d="M23 287Q24 290 25 295T30 317T40 348T55 381T75 411T101 433T134 442Q209 442 230 378L240 387Q302 442 358 442Q423 442 460 395T497 281Q497 173 421 82T249 -10Q227 -10 210 -4Q199 1 187 11T168 28L161 36Q160 35 139 -51T118 -138Q118 -144 126 -145T163 -148H188Q194 -155 194 -157T191 -175Q188 -187 185 -190T172 -194Q170 -194 161 -194T127 -193T65 -192Q-5 -192 -24 -194H-32Q-39 -187 -39 -183Q-37 -156 -26 -148H-6Q28 -147 33 -136Q36 -130 94 103T155 350Q156 355 156 364Q156 405 131 405Q109 405 94 377T71 316T59 280Q57 278 43 278H29Q23 284 23 287ZM178 102Q200 26 252 26Q282 26 310 49T356 107Q374 141 392 215T411 325V331Q411 405 350 405Q339 405 328 402T306 393T286 380T269 365T254 350T243 336T235 326L232 322Q232 321 229 308T218 264T204 212Q178 106 178 102Z"></path><path id="MJMAINB-6E" stroke-width="1" d="M40 442Q217 450 218 450H224V407L225 365Q233 378 245 391T289 422T362 448Q374 450 398 450Q428 450 448 447T491 434T529 402T551 346Q553 335 554 198V62H623V0H614Q596 3 489 3Q374 3 365 0H356V62H425V194V275Q425 348 416 373T371 399Q326 399 288 370T238 290Q236 281 235 171V62H304V0H295Q277 3 171 3Q64 3 46 0H37V62H106V210V303Q106 353 104 363T91 376Q77 380 50 380H37V442H40Z"></path><path id="MJMAINB-65" stroke-width="1" d="M32 225Q32 332 102 392T272 452H283Q382 452 436 401Q494 343 494 243Q494 226 486 222T440 217Q431 217 394 217T327 218H175V209Q175 177 179 154T196 107T236 69T306 50Q312 49 323 49Q376 49 410 85Q421 99 427 111T434 127T442 133T463 135H468Q494 135 494 117Q494 110 489 97T468 66T431 32T373 5T292 -6Q181 -6 107 55T32 225ZM383 276Q377 346 348 374T280 402Q253 402 230 390T195 357Q179 331 176 279V266H383V276Z"></path><path id="MJMAINB-74" stroke-width="1" d="M272 49Q320 49 320 136V145V177H382V143Q382 106 380 99Q374 62 349 36T285 -2L272 -5H247Q173 -5 134 27Q109 46 102 74T94 160Q94 171 94 199T95 245V382H21V433H25Q58 433 90 456Q121 479 140 523T162 621V635H224V444H363V382H224V239V207V149Q224 98 228 81T249 55Q261 49 272 49Z"></path><path id="MJMATHI-65" stroke-width="1" d="M39 168Q39 225 58 272T107 350T174 402T244 433T307 442H310Q355 442 388 420T421 355Q421 265 310 237Q261 224 176 223Q139 223 138 221Q138 219 132 186T125 128Q125 81 146 54T209 26T302 45T394 111Q403 121 406 121Q410 121 419 112T429 98T420 82T390 55T344 24T281 -1T205 -11Q126 -11 83 42T39 168ZM373 353Q367 405 305 405Q272 405 244 391T199 357T170 316T154 280T149 261Q149 260 169 260Q282 260 327 284T373 353Z"></path><path id="MJMATHI-66" stroke-width="1" d="M118 -162Q120 -162 124 -164T135 -167T147 -168Q160 -168 171 -155T187 -126Q197 -99 221 27T267 267T289 382V385H242Q195 385 192 387Q188 390 188 397L195 425Q197 430 203 430T250 431Q298 431 298 432Q298 434 307 482T319 540Q356 705 465 705Q502 703 526 683T550 630Q550 594 529 578T487 561Q443 561 443 603Q443 622 454 636T478 657L487 662Q471 668 457 668Q445 668 434 658T419 630Q412 601 403 552T387 469T380 433Q380 431 435 431Q480 431 487 430T498 424Q499 420 496 407T491 391Q489 386 482 386T428 385H372L349 263Q301 15 282 -47Q255 -132 212 -173Q175 -205 139 -205Q107 -205 81 -186T55 -132Q55 -95 76 -78T118 -61Q162 -61 162 -103Q162 -122 151 -136T127 -157L118 -162Z"></path><path id="MJMAIN-2202" stroke-width="1" d="M202 508Q179 508 169 520T158 547Q158 557 164 577T185 624T230 675T301 710L333 715H345Q378 715 384 714Q447 703 489 661T549 568T566 457Q566 362 519 240T402 53Q321 -22 223 -22Q123 -22 73 56Q42 102 42 148V159Q42 276 129 370T322 465Q383 465 414 434T455 367L458 378Q478 461 478 515Q478 603 437 639T344 676Q266 676 223 612Q264 606 264 572Q264 547 246 528T202 508ZM430 306Q430 372 401 400T333 428Q270 428 222 382Q197 354 183 323T150 221Q132 149 132 116Q132 21 232 21Q244 21 250 22Q327 35 374 112Q389 137 409 196T430 306Z"></path><path id="MJMATHI-45" stroke-width="1" d="M492 213Q472 213 472 226Q472 230 477 250T482 285Q482 316 461 323T364 330H312Q311 328 277 192T243 52Q243 48 254 48T334 46Q428 46 458 48T518 61Q567 77 599 117T670 248Q680 270 683 272Q690 274 698 274Q718 274 718 261Q613 7 608 2Q605 0 322 0H133Q31 0 31 11Q31 13 34 25Q38 41 42 43T65 46Q92 46 125 49Q139 52 144 61Q146 66 215 342T285 622Q285 629 281 629Q273 632 228 634H197Q191 640 191 642T193 659Q197 676 203 680H757Q764 676 764 669Q764 664 751 557T737 447Q735 440 717 440H705Q698 445 698 453L701 476Q704 500 704 528Q704 558 697 578T678 609T643 625T596 632T532 634H485Q397 633 392 631Q388 629 386 622Q385 619 355 499T324 377Q347 376 372 376H398Q464 376 489 391T534 472Q538 488 540 490T557 493Q562 493 565 493T570 492T572 491T574 487T577 483L544 351Q511 218 508 216Q505 213 492 213Z"></path><path id="MJMAIN-2E" stroke-width="1" d="M78 60Q78 84 95 102T138 120Q162 120 180 104T199 61Q199 36 182 18T139 0T96 17T78 60Z"></path><path id="MJSZ4-23A1" stroke-width="1" d="M319 -645V1154H666V1070H403V-645H319Z"></path><path id="MJSZ4-23A3" stroke-width="1" d="M319 -644V1155H403V-560H666V-644H319Z"></path><path id="MJSZ4-23A2" stroke-width="1" d="M319 0V602H403V0H319Z"></path><path id="MJSZ4-23A4" stroke-width="1" d="M0 1070V1154H347V-645H263V1070H0Z"></path><path id="MJSZ4-23A6" stroke-width="1" d="M263 -560V1155H347V-644H0V-560H263Z"></path><path id="MJSZ4-23A5" stroke-width="1" d="M263 0V602H347V0H263Z"></path><path id="MJMATHI-77" stroke-width="1" d="M580 385Q580 406 599 424T641 443Q659 443 674 425T690 368Q690 339 671 253Q656 197 644 161T609 80T554 12T482 -11Q438 -11 404 5T355 48Q354 47 352 44Q311 -11 252 -11Q226 -11 202 -5T155 14T118 53T104 116Q104 170 138 262T173 379Q173 380 173 381Q173 390 173 393T169 400T158 404H154Q131 404 112 385T82 344T65 302T57 280Q55 278 41 278H27Q21 284 21 287Q21 293 29 315T52 366T96 418T161 441Q204 441 227 416T250 358Q250 340 217 250T184 111Q184 65 205 46T258 26Q301 26 334 87L339 96V119Q339 122 339 128T340 136T341 143T342 152T345 165T348 182T354 206T362 238T373 281Q402 395 406 404Q419 431 449 431Q468 431 475 421T483 402Q483 389 454 274T422 142Q420 131 420 107V100Q420 85 423 71T442 42T487 26Q558 26 600 148Q609 171 620 213T632 273Q632 306 619 325T593 357T580 385Z"></path><path id="MJMATHI-63" stroke-width="1" d="M34 159Q34 268 120 355T306 442Q362 442 394 418T427 355Q427 326 408 306T360 285Q341 285 330 295T319 325T330 359T352 380T366 386H367Q367 388 361 392T340 400T306 404Q276 404 249 390Q228 381 206 359Q162 315 142 235T121 119Q121 73 147 50Q169 26 205 26H209Q321 26 394 111Q403 121 406 121Q410 121 419 112T429 98T420 83T391 55T346 25T282 0T202 -11Q127 -11 81 37T34 159Z"></path><path id="MJMATHI-62" stroke-width="1" d="M73 647Q73 657 77 670T89 683Q90 683 161 688T234 694Q246 694 246 685T212 542Q204 508 195 472T180 418L176 399Q176 396 182 402Q231 442 283 442Q345 442 383 396T422 280Q422 169 343 79T173 -11Q123 -11 82 27T40 150V159Q40 180 48 217T97 414Q147 611 147 623T109 637Q104 637 101 637H96Q86 637 83 637T76 640T73 647ZM336 325V331Q336 405 275 405Q258 405 240 397T207 376T181 352T163 330L157 322L136 236Q114 150 114 114Q114 66 138 42Q154 26 178 26Q211 26 245 58Q270 81 285 114T318 219Q336 291 336 325Z"></path><path id="MJMATHI-69" stroke-width="1" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369T98 420T158 442Q197 442 223 419T250 357Q250 340 236 301T196 196T154 83Q149 61 149 51Q149 26 166 26Q175 26 185 29T208 43T235 78T260 137Q263 149 265 151T282 153Q302 153 302 143Q302 135 293 112T268 61T223 11T161 -11Q129 -11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281T56 279T53 278T49 278T41 278H27Q21 284 21 287Z"></path><path id="MJMATHI-6A" stroke-width="1" d="M297 596Q297 627 318 644T361 661Q378 661 389 651T403 623Q403 595 384 576T340 557Q322 557 310 567T297 596ZM288 376Q288 405 262 405Q240 405 220 393T185 362T161 325T144 293L137 279Q135 278 121 278H107Q101 284 101 286T105 299Q126 348 164 391T252 441Q253 441 260 441T272 442Q296 441 316 432Q341 418 354 401T367 348V332L318 133Q267 -67 264 -75Q246 -125 194 -164T75 -204Q25 -204 7 -183T-12 -137Q-12 -110 7 -91T53 -71Q70 -71 82 -81T95 -112Q95 -148 63 -167Q69 -168 77 -168Q111 -168 139 -140T182 -74L193 -32Q204 11 219 72T251 197T278 308T289 365Q289 372 288 376Z"></path><path id="MJSZ2-2211" stroke-width="1" d="M60 948Q63 950 665 950H1267L1325 815Q1384 677 1388 669H1348L1341 683Q1320 724 1285 761Q1235 809 1174 838T1033 881T882 898T699 902H574H543H251L259 891Q722 258 724 252Q725 250 724 246Q721 243 460 -56L196 -356Q196 -357 407 -357Q459 -357 548 -357T676 -358Q812 -358 896 -353T1063 -332T1204 -283T1307 -196Q1328 -170 1348 -124H1388Q1388 -125 1381 -145T1356 -210T1325 -294L1267 -449L666 -450Q64 -450 61 -448Q55 -446 55 -439Q55 -437 57 -433L590 177Q590 178 557 222T452 366T322 544L56 909L55 924Q55 945 60 948Z"></path><path id="MJMAIN-34" stroke-width="1" d="M462 0Q444 3 333 3Q217 3 199 0H190V46H221Q241 46 248 46T265 48T279 53T286 61Q287 63 287 115V165H28V211L179 442Q332 674 334 675Q336 677 355 677H373L379 671V211H471V165H379V114Q379 73 379 66T385 54Q393 47 442 46H471V0H462ZM293 211V545L74 212L183 211H293Z"></path><path id="MJMATHI-55" stroke-width="1" d="M107 637Q73 637 71 641Q70 643 70 649Q70 673 81 682Q83 683 98 683Q139 681 234 681Q268 681 297 681T342 682T362 682Q378 682 378 672Q378 670 376 658Q371 641 366 638H364Q362 638 359 638T352 638T343 637T334 637Q295 636 284 634T266 623Q265 621 238 518T184 302T154 169Q152 155 152 140Q152 86 183 55T269 24Q336 24 403 69T501 205L552 406Q599 598 599 606Q599 633 535 637Q511 637 511 648Q511 650 513 660Q517 676 519 679T529 683Q532 683 561 682T645 680Q696 680 723 681T752 682Q767 682 767 672Q767 650 759 642Q756 637 737 637Q666 633 648 597Q646 592 598 404Q557 235 548 205Q515 105 433 42T263 -22Q171 -22 116 34T60 167V183Q60 201 115 421Q164 622 164 628Q164 635 107 637Z"></path><path id="MJMATHI-75" stroke-width="1" d="M21 287Q21 295 30 318T55 370T99 420T158 442Q204 442 227 417T250 358Q250 340 216 246T182 105Q182 62 196 45T238 27T291 44T328 78L339 95Q341 99 377 247Q407 367 413 387T427 416Q444 431 463 431Q480 431 488 421T496 402L420 84Q419 79 419 68Q419 43 426 35T447 26Q469 29 482 57T512 145Q514 153 532 153Q551 153 551 144Q550 139 549 130T540 98T523 55T498 17T462 -8Q454 -10 438 -10Q372 -10 347 46Q345 45 336 36T318 21T296 6T267 -6T233 -11Q189 -11 155 7Q103 38 103 113Q103 170 138 262T173 379Q173 380 173 381Q173 390 173 393T169 400T158 404H154Q131 404 112 385T82 344T65 302T57 280Q55 278 41 278H27Q21 284 21 287Z"></path><path id="MJMATHI-54" stroke-width="1" d="M40 437Q21 437 21 445Q21 450 37 501T71 602L88 651Q93 669 101 677H569H659Q691 677 697 676T704 667Q704 661 687 553T668 444Q668 437 649 437Q640 437 637 437T631 442L629 445Q629 451 635 490T641 551Q641 586 628 604T573 629Q568 630 515 631Q469 631 457 630T439 622Q438 621 368 343T298 60Q298 48 386 46Q418 46 427 45T436 36Q436 31 433 22Q429 4 424 1L422 0Q419 0 415 0Q410 0 363 1T228 2Q99 2 64 0H49Q43 6 43 9T45 27Q49 40 55 46H83H94Q174 46 189 55Q190 56 191 56Q196 59 201 76T241 233Q258 301 269 344Q339 619 339 625Q339 630 310 630H279Q212 630 191 624Q146 614 121 583T67 467Q60 445 57 441T43 437H40Z"></path><path id="MJMAIN-35" stroke-width="1" d="M164 157Q164 133 148 117T109 101H102Q148 22 224 22Q294 22 326 82Q345 115 345 210Q345 313 318 349Q292 382 260 382H254Q176 382 136 314Q132 307 129 306T114 304Q97 304 95 310Q93 314 93 485V614Q93 664 98 664Q100 666 102 666Q103 666 123 658T178 642T253 634Q324 634 389 662Q397 666 402 666Q410 666 410 648V635Q328 538 205 538Q174 538 149 544L139 546V374Q158 388 169 396T205 412T256 420Q337 420 393 355T449 201Q449 109 385 44T229 -22Q148 -22 99 32T50 154Q50 178 61 192T84 210T107 214Q132 214 148 197T164 157Z"></path><path id="MJMAIN-36" stroke-width="1" d="M42 313Q42 476 123 571T303 666Q372 666 402 630T432 550Q432 525 418 510T379 495Q356 495 341 509T326 548Q326 592 373 601Q351 623 311 626Q240 626 194 566Q147 500 147 364L148 360Q153 366 156 373Q197 433 263 433H267Q313 433 348 414Q372 400 396 374T435 317Q456 268 456 210V192Q456 169 451 149Q440 90 387 34T253 -22Q225 -22 199 -14T143 16T92 75T56 172T42 313ZM257 397Q227 397 205 380T171 335T154 278T148 216Q148 133 160 97T198 39Q222 21 251 21Q302 21 329 59Q342 77 347 104T352 209Q352 289 347 316T329 361Q302 397 257 397Z"></path><path id="MJMAIN-37" stroke-width="1" d="M55 458Q56 460 72 567L88 674Q88 676 108 676H128V672Q128 662 143 655T195 646T364 644H485V605L417 512Q408 500 387 472T360 435T339 403T319 367T305 330T292 284T284 230T278 162T275 80Q275 66 275 52T274 28V19Q270 2 255 -10T221 -22Q210 -22 200 -19T179 0T168 40Q168 198 265 368Q285 400 349 489L395 552H302Q128 552 119 546Q113 543 108 522T98 479L95 458V455H55V458Z"></path><path id="MJMAIN-38" stroke-width="1" d="M70 417T70 494T124 618T248 666Q319 666 374 624T429 515Q429 485 418 459T392 417T361 389T335 371T324 363L338 354Q352 344 366 334T382 323Q457 264 457 174Q457 95 399 37T249 -22Q159 -22 101 29T43 155Q43 263 172 335L154 348Q133 361 127 368Q70 417 70 494ZM286 386L292 390Q298 394 301 396T311 403T323 413T334 425T345 438T355 454T364 471T369 491T371 513Q371 556 342 586T275 624Q268 625 242 625Q201 625 165 599T128 534Q128 511 141 492T167 463T217 431Q224 426 228 424L286 386ZM250 21Q308 21 350 55T392 137Q392 154 387 169T375 194T353 216T330 234T301 253T274 270Q260 279 244 289T218 306L210 311Q204 311 181 294T133 239T107 157Q107 98 150 60T250 21Z"></path><path id="MJMAIN-2218" stroke-width="1" d="M55 251Q55 328 112 386T249 444T386 388T444 249Q444 171 388 113T250 55Q170 55 113 112T55 251ZM245 403Q188 403 142 361T96 250Q96 183 141 140T250 96Q284 96 313 109T354 135T375 160Q403 197 403 250Q403 313 360 358T245 403Z"></path><path id="MJMAIN-2032" stroke-width="1" d="M79 43Q73 43 52 49T30 61Q30 68 85 293T146 528Q161 560 198 560Q218 560 240 545T262 501Q262 496 260 486Q259 479 173 263T84 45T79 43Z"></path><path id="MJSZ4-5B" stroke-width="1" d="M269 -1249V1750H577V1677H342V-1176H577V-1249H269Z"></path><path id="MJSZ4-5D" stroke-width="1" d="M5 1677V1750H313V-1249H5V-1176H240V1677H5Z"></path><path id="MJMAIN-39" stroke-width="1" d="M352 287Q304 211 232 211Q154 211 104 270T44 396Q42 412 42 436V444Q42 537 111 606Q171 666 243 666Q245 666 249 666T257 665H261Q273 665 286 663T323 651T370 619T413 560Q456 472 456 334Q456 194 396 97Q361 41 312 10T208 -22Q147 -22 108 7T68 93T121 149Q143 149 158 135T173 96Q173 78 164 65T148 49T135 44L131 43Q131 41 138 37T164 27T206 22H212Q272 22 313 86Q352 142 352 280V287ZM244 248Q292 248 321 297T351 430Q351 508 343 542Q341 552 337 562T323 588T293 615T246 625Q208 625 181 598Q160 576 154 546T147 441Q147 358 152 329T172 282Q197 248 244 248Z"></path><path id="MJMAIN-30" stroke-width="1" d="M96 585Q152 666 249 666Q297 666 345 640T423 548Q460 465 460 320Q460 165 417 83Q397 41 362 16T301 -15T250 -22Q224 -22 198 -16T137 16T82 83Q39 165 39 320Q39 494 96 585ZM321 597Q291 629 250 629Q208 629 178 597Q153 571 145 525T137 333Q137 175 145 125T181 46Q209 16 250 16Q290 16 318 46Q347 76 354 130T362 333Q362 478 354 524T321 597Z"></path><path id="MJMATHI-6C" stroke-width="1" d="M117 59Q117 26 142 26Q179 26 205 131Q211 151 215 152Q217 153 225 153H229Q238 153 241 153T246 151T248 144Q247 138 245 128T234 90T214 43T183 6T137 -11Q101 -11 70 11T38 85Q38 97 39 102L104 360Q167 615 167 623Q167 626 166 628T162 632T157 634T149 635T141 636T132 637T122 637Q112 637 109 637T101 638T95 641T94 647Q94 649 96 661Q101 680 107 682T179 688Q194 689 213 690T243 693T254 694Q266 694 266 686Q266 675 193 386T118 83Q118 81 118 75T117 65V59Z"></path><path id="MJMATHI-6D" stroke-width="1" d="M21 287Q22 293 24 303T36 341T56 388T88 425T132 442T175 435T205 417T221 395T229 376L231 369Q231 367 232 367L243 378Q303 442 384 442Q401 442 415 440T441 433T460 423T475 411T485 398T493 385T497 373T500 364T502 357L510 367Q573 442 659 442Q713 442 746 415T780 336Q780 285 742 178T704 50Q705 36 709 31T724 26Q752 26 776 56T815 138Q818 149 821 151T837 153Q857 153 857 145Q857 144 853 130Q845 101 831 73T785 17T716 -10Q669 -10 648 17T627 73Q627 92 663 193T700 345Q700 404 656 404H651Q565 404 506 303L499 291L466 157Q433 26 428 16Q415 -11 385 -11Q372 -11 364 -4T353 8T350 18Q350 29 384 161L420 307Q423 322 423 345Q423 404 379 404H374Q288 404 229 303L222 291L189 157Q156 26 151 16Q138 -11 108 -11Q95 -11 87 -5T76 7T74 17Q74 30 112 181Q151 335 151 342Q154 357 154 369Q154 405 129 405Q107 405 92 377T69 316T57 280Q55 278 41 278H27Q21 284 21 287Z"></path><path id="MJMAIN-2190" stroke-width="1" d="M944 261T944 250T929 230H165Q167 228 182 216T211 189T244 152T277 96T303 25Q308 7 308 0Q308 -11 288 -11Q281 -11 278 -11T272 -7T267 2T263 21Q245 94 195 151T73 236Q58 242 55 247Q55 254 59 257T73 264Q121 283 158 314T215 375T247 434T264 480L267 497Q269 503 270 505T275 509T288 511Q308 511 308 500Q308 493 303 475Q293 438 278 406T246 352T215 315T185 287T165 270H929Q944 261 944 250Z"></path><path id="MJMATHI-3B7" stroke-width="1" d="M21 287Q22 290 23 295T28 317T38 348T53 381T73 411T99 433T132 442Q156 442 175 435T205 417T221 395T229 376L231 369Q231 367 232 367L243 378Q304 442 382 442Q436 442 469 415T503 336V326Q503 302 439 53Q381 -182 377 -189Q364 -216 332 -216Q319 -216 310 -208T299 -186Q299 -177 358 57L420 307Q423 322 423 345Q423 404 379 404H374Q288 404 229 303L222 291L189 157Q156 26 151 16Q138 -11 108 -11Q95 -11 87 -5T76 7T74 17Q74 30 114 189T154 366Q154 405 128 405Q107 405 92 377T68 316T57 280Q55 278 41 278H27Q21 284 21 287Z"></path><path id="MJMATHI-4A" stroke-width="1" d="M447 625Q447 637 354 637H329Q323 642 323 645T325 664Q329 677 335 683H352Q393 681 498 681Q541 681 568 681T605 682T619 682Q633 682 633 672Q633 670 630 658Q626 642 623 640T604 637Q552 637 545 623Q541 610 483 376Q420 128 419 127Q397 64 333 21T195 -22Q137 -22 97 8T57 88Q57 130 80 152T132 174Q177 174 182 130Q182 98 164 80T123 56Q115 54 115 53T122 44Q148 15 197 15Q235 15 271 47T324 130Q328 142 387 380T447 625Z"></path><path id="MJMATHI-3B8" stroke-width="1" d="M35 200Q35 302 74 415T180 610T319 704Q320 704 327 704T339 705Q393 701 423 656Q462 596 462 495Q462 380 417 261T302 66T168 -10H161Q125 -10 99 10T60 63T41 130T35 200ZM383 566Q383 668 330 668Q294 668 260 623T204 521T170 421T157 371Q206 370 254 370L351 371Q352 372 359 404T375 484T383 566ZM113 132Q113 26 166 26Q181 26 198 36T239 74T287 161T335 307L340 324H145Q145 321 136 286T120 208T113 132Z"></path><path id="MJMATHI-78" stroke-width="1" d="M52 289Q59 331 106 386T222 442Q257 442 286 424T329 379Q371 442 430 442Q467 442 494 420T522 361Q522 332 508 314T481 292T458 288Q439 288 427 299T415 328Q415 374 465 391Q454 404 425 404Q412 404 406 402Q368 386 350 336Q290 115 290 78Q290 50 306 38T341 26Q378 26 414 59T463 140Q466 150 469 151T485 153H489Q504 153 504 145Q504 144 502 134Q486 77 440 33T333 -11Q263 -11 227 52Q186 -10 133 -10H127Q78 -10 57 16T35 71Q35 103 54 123T99 143Q142 143 142 101Q142 81 130 66T107 46T94 41L91 40Q91 39 97 36T113 29T132 26Q168 26 194 71Q203 87 217 139T245 247T261 313Q266 340 266 352Q266 380 251 392T217 404Q177 404 142 372T93 290Q91 281 88 280T72 278H58Q52 284 52 289Z"></path><path id="MJMAIN-2C" stroke-width="1" d="M78 35T78 60T94 103T137 121Q165 121 187 96T210 8Q210 -27 201 -60T180 -117T154 -158T130 -185T117 -194Q113 -194 104 -185T95 -172Q95 -168 106 -156T131 -126T157 -76T173 -3V9L172 8Q170 7 167 6T161 3T152 1T140 0Q113 0 96 17Z"></path><path id="MJMATHI-79" stroke-width="1" d="M21 287Q21 301 36 335T84 406T158 442Q199 442 224 419T250 355Q248 336 247 334Q247 331 231 288T198 191T182 105Q182 62 196 45T238 27Q261 27 281 38T312 61T339 94Q339 95 344 114T358 173T377 247Q415 397 419 404Q432 431 462 431Q475 431 483 424T494 412T496 403Q496 390 447 193T391 -23Q363 -106 294 -155T156 -205Q111 -205 77 -183T43 -117Q43 -95 50 -80T69 -58T89 -48T106 -45Q150 -45 150 -87Q150 -107 138 -122T115 -142T102 -147L99 -148Q101 -153 118 -160T152 -167H160Q177 -167 186 -165Q219 -156 247 -127T290 -65T313 -9T321 21L315 17Q309 13 296 6T270 -6Q250 -11 231 -11Q185 -11 150 11T104 82Q103 89 103 113Q103 170 138 262T173 379Q173 380 173 381Q173 390 173 393T169 400T158 404H154Q131 404 112 385T82 344T65 302T57 280Q55 278 41 278H27Q21 284 21 287Z"></path><path id="MJMAIN-2208" stroke-width="1" d="M84 250Q84 372 166 450T360 539Q361 539 377 539T419 540T469 540H568Q583 532 583 520Q583 511 570 501L466 500Q355 499 329 494Q280 482 242 458T183 409T147 354T129 306T124 272V270H568Q583 262 583 250T568 230H124V228Q124 207 134 177T167 112T231 48T328 7Q355 1 466 0H570Q583 -10 583 -20Q583 -32 568 -40H471Q464 -40 446 -40T417 -41Q262 -41 172 45Q84 127 84 250Z"></path><path id="MJMATHI-41" stroke-width="1" d="M208 74Q208 50 254 46Q272 46 272 35Q272 34 270 22Q267 8 264 4T251 0Q249 0 239 0T205 1T141 2Q70 2 50 0H42Q35 7 35 11Q37 38 48 46H62Q132 49 164 96Q170 102 345 401T523 704Q530 716 547 716H555H572Q578 707 578 706L606 383Q634 60 636 57Q641 46 701 46Q726 46 726 36Q726 34 723 22Q720 7 718 4T704 0Q701 0 690 0T651 1T578 2Q484 2 455 0H443Q437 6 437 9T439 27Q443 40 445 43L449 46H469Q523 49 533 63L521 213H283L249 155Q208 86 208 74ZM516 260Q516 271 504 416T490 562L463 519Q447 492 400 412L310 260L413 259Q516 259 516 260Z"></path><path id="MJMATHI-73" stroke-width="1" d="M131 289Q131 321 147 354T203 415T300 442Q362 442 390 415T419 355Q419 323 402 308T364 292Q351 292 340 300T328 326Q328 342 337 354T354 372T367 378Q368 378 368 379Q368 382 361 388T336 399T297 405Q249 405 227 379T204 326Q204 301 223 291T278 274T330 259Q396 230 396 163Q396 135 385 107T352 51T289 7T195 -10Q118 -10 86 19T53 87Q53 126 74 143T118 160Q133 160 146 151T160 120Q160 94 142 76T111 58Q109 57 108 57T107 55Q108 52 115 47T146 34T201 27Q237 27 263 38T301 66T318 97T323 122Q323 150 302 164T254 181T195 196T148 231Q131 256 131 289Z"></path><path id="MJMAIN-394" stroke-width="1" d="M51 0Q46 4 46 7Q46 9 215 357T388 709Q391 716 416 716Q439 716 444 709Q447 705 616 357T786 7Q786 4 781 0H51ZM507 344L384 596L137 92L383 91H630Q630 93 507 344Z"></path><path id="MJMAIN-2212" stroke-width="1" d="M84 237T84 250T98 270H679Q694 262 694 250T679 230H98Q84 237 84 250Z"></path><path id="MJMATHI-72" stroke-width="1" d="M21 287Q22 290 23 295T28 317T38 348T53 381T73 411T99 433T132 442Q161 442 183 430T214 408T225 388Q227 382 228 382T236 389Q284 441 347 441H350Q398 441 422 400Q430 381 430 363Q430 333 417 315T391 292T366 288Q346 288 334 299T322 328Q322 376 378 392Q356 405 342 405Q286 405 239 331Q229 315 224 298T190 165Q156 25 151 16Q138 -11 108 -11Q95 -11 87 -5T76 7T74 17Q74 30 114 189T154 366Q154 405 128 405Q107 405 92 377T68 316T57 280Q55 278 41 278H27Q21 284 21 287Z"></path><path id="MJMATHI-67" stroke-width="1" d="M311 43Q296 30 267 15T206 0Q143 0 105 45T66 160Q66 265 143 353T314 442Q361 442 401 394L404 398Q406 401 409 404T418 412T431 419T447 422Q461 422 470 413T480 394Q480 379 423 152T363 -80Q345 -134 286 -169T151 -205Q10 -205 10 -137Q10 -111 28 -91T74 -71Q89 -71 102 -80T116 -111Q116 -121 114 -130T107 -144T99 -154T92 -162L90 -164H91Q101 -167 151 -167Q189 -167 211 -155Q234 -144 254 -122T282 -75Q288 -56 298 -13Q311 35 311 43ZM384 328L380 339Q377 350 375 354T369 368T359 382T346 393T328 402T306 405Q262 405 221 352Q191 313 171 233T151 117Q151 38 213 38Q269 38 323 108L331 118L384 328Z"></path><path id="MJMATHI-4E" stroke-width="1" d="M234 637Q231 637 226 637Q201 637 196 638T191 649Q191 676 202 682Q204 683 299 683Q376 683 387 683T401 677Q612 181 616 168L670 381Q723 592 723 606Q723 633 659 637Q635 637 635 648Q635 650 637 660Q641 676 643 679T653 683Q656 683 684 682T767 680Q817 680 843 681T873 682Q888 682 888 672Q888 650 880 642Q878 637 858 637Q787 633 769 597L620 7Q618 0 599 0Q585 0 582 2Q579 5 453 305L326 604L261 344Q196 88 196 79Q201 46 268 46H278Q284 41 284 38T282 19Q278 6 272 0H259Q228 2 151 2Q123 2 100 2T63 2T46 1Q31 1 31 10Q31 14 34 26T39 40Q41 46 62 46Q130 49 150 85Q154 91 221 362L289 634Q287 635 234 637Z"></path><path id="MJMAINB-31" stroke-width="1" d="M481 0L294 3Q136 3 109 0H96V62H227V304Q227 546 225 546Q169 529 97 529H80V591H97Q231 591 308 647L319 655H333Q355 655 359 644Q361 640 361 351V62H494V0H481Z"></path><path id="MJMAIN-7B" stroke-width="1" d="M434 -231Q434 -244 428 -250H410Q281 -250 230 -184Q225 -177 222 -172T217 -161T213 -148T211 -133T210 -111T209 -84T209 -47T209 0Q209 21 209 53Q208 142 204 153Q203 154 203 155Q189 191 153 211T82 231Q71 231 68 234T65 250T68 266T82 269Q116 269 152 289T203 345Q208 356 208 377T209 529V579Q209 634 215 656T244 698Q270 724 324 740Q361 748 377 749Q379 749 390 749T408 750H428Q434 744 434 732Q434 719 431 716Q429 713 415 713Q362 710 332 689T296 647Q291 634 291 499V417Q291 370 288 353T271 314Q240 271 184 255L170 250L184 245Q202 239 220 230T262 196T290 137Q291 131 291 1Q291 -134 296 -147Q306 -174 339 -192T415 -213Q429 -213 431 -216Q434 -219 434 -231Z"></path><path id="MJMAIN-2209" stroke-width="1" d="M196 25Q84 109 84 250Q84 372 166 450T360 539Q361 539 375 539T413 540T460 540L547 707Q550 716 563 716Q570 716 575 712T581 703T583 696T505 540H568Q583 532 583 520Q583 511 570 501L484 500L366 270H568Q583 262 583 250T568 230H346L247 38Q284 16 328 7Q355 1 466 0H570Q583 -10 583 -20Q583 -32 568 -40H471Q464 -40 447 -40T419 -41Q304 -41 228 3Q117 -211 115 -212Q111 -215 104 -215T92 -212T86 -204T84 -197Q84 -190 89 -183L196 25ZM214 61L301 230H124V228Q124 196 147 147T214 61ZM321 270L440 500Q353 499 329 494Q280 482 242 458T183 409T147 354T129 306T124 272V270H321Z"></path><path id="MJMAIN-7D" stroke-width="1" d="M65 731Q65 745 68 747T88 750Q171 750 216 725T279 670Q288 649 289 635T291 501Q292 362 293 357Q306 312 345 291T417 269Q428 269 431 266T434 250T431 234T417 231Q380 231 345 210T298 157Q293 143 292 121T291 -28V-79Q291 -134 285 -156T256 -198Q202 -250 89 -250Q71 -250 68 -247T65 -230Q65 -224 65 -223T66 -218T69 -214T77 -213Q91 -213 108 -210T146 -200T183 -177T207 -139Q208 -134 209 3L210 139Q223 196 280 230Q315 247 330 250Q305 257 280 270Q225 304 212 352L210 362L209 498Q208 635 207 640Q195 680 154 696T77 713Q68 713 67 716T65 731Z"></path><path id="MJMATHI-6F" stroke-width="1" d="M201 -11Q126 -11 80 38T34 156Q34 221 64 279T146 380Q222 441 301 441Q333 441 341 440Q354 437 367 433T402 417T438 387T464 338T476 268Q476 161 390 75T201 -11ZM121 120Q121 70 147 48T206 26Q250 26 289 58T351 142Q360 163 374 216T388 308Q388 352 370 375Q346 405 306 405Q243 405 195 347Q158 303 140 230T121 120Z"></path><path id="MJMAINB-78" stroke-width="1" d="M227 0Q212 3 121 3Q40 3 28 0H21V62H117L245 213L109 382H26V444H34Q49 441 143 441Q247 441 265 444H274V382H246L281 339Q315 297 316 297Q320 297 354 341L389 382H352V444H360Q375 441 466 441Q547 441 559 444H566V382H471L355 246L504 63L545 62H586V0H578Q563 3 469 3Q365 3 347 0H338V62H366Q366 63 326 112T285 163L198 63L217 62H235V0H227Z"></path><path id="MJMAINB-79" stroke-width="1" d="M84 -102Q84 -110 87 -119T102 -138T133 -149Q148 -148 162 -143T186 -131T206 -114T222 -95T234 -76T243 -59T249 -45T252 -37L269 0L96 382H26V444H34Q49 441 146 441Q252 441 270 444H279V382H255Q232 382 232 380L337 151L442 382H394V444H401Q413 441 495 441Q568 441 574 444H580V382H510L406 152Q298 -84 297 -87Q269 -139 225 -169T131 -200Q85 -200 54 -172T23 -100Q23 -64 44 -50T87 -35Q111 -35 130 -50T152 -92V-100H84V-102Z"></path></defs></svg></div><div id="MathJax_Message" style="display: none;"></div><div class="pace  pace-inactive"><div class="pace-progress" data-progress-text="100%" data-progress="99" style="width: 100%;">
  <div class="pace-progress-inner"></div>
</div>
<div class="pace-activity"></div></div>

    <div id="global-prompt-alert" class="hide alert alert-warning">
        <span id="global-prompt-message"></span>
        <a id="close-global-prompt-alert" href="https://zybuluo.com/hanbingtao/note/626300">[关闭]</a>
    </div>

    <!-- zybuluo's body -->
    







<!-- mdeditor's body -->






<div id="editor-reader-full" class="editor-reader-full-shown" style="position: static; padding-right: 75px;">
    <div id="reader-full-topInfo" class="reader-full-topInfo-shown">
        <span>
            <code>@hanbingtao</code>
        </span>
        <code><span class="article-updated-date">2017-08-29 23:41</span></code>
        <code><span>字数 </span><span class="article-characters">14413</span></code>
        <code><span>阅读 </span><span class="article-read">64866</span></code>
    </div>
    <div id="wmd-preview" class="wmd-preview wmd-preview-full-reader" data-medium-element="true"><div class="md-section-divider"></div><div class="md-section-divider"></div><h1 data-anchor-id="ti22" id="零基础入门深度学习7-递归神经网络">零基础入门深度学习(7) - 递归神经网络</h1><p data-anchor-id="y9eo"><code>机器学习</code> <code>深度学习入门</code></p><hr><p data-anchor-id="cdsk"><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-06627c71f0d8c0dc.jpg" alt="" title="点击查看大图"></p><blockquote data-anchor-id="tdyo" class="white-blockquote">
  <p>无论即将到来的是大数据时代还是人工智能时代，亦或是传统行业使用人工智能在云上处理大数据的时代，作为一个有理想有追求的程序员，不懂深度学习（Deep Learning）这个超热的技术，会不会感觉马上就out了？现在救命稻草来了，《零基础入门深度学习》系列文章旨在讲帮助爱编程的你从零基础达到入门级水平。零基础意味着你不需要太多的数学知识，只要会写程序就行了，没错，这是专门为程序员写的文章。虽然文中会有很多公式你也许看不懂，但同时也会有更多的代码，程序员的你一定能看懂的（我周围是一群狂热的Clean Code程序员，所以我写的代码也不会很差）。</p>
</blockquote><div class="md-section-divider"></div><h2 data-anchor-id="zy0s" id="文章列表">文章列表</h2><p data-anchor-id="rb6q"><a href="https://www.zybuluo.com/hanbingtao/note/433855" target="_blank">零基础入门深度学习(1) - 感知器</a> <br>
<a href="https://www.zybuluo.com/hanbingtao/note/448086" target="_blank">零基础入门深度学习(2) - 线性单元和梯度下降</a> <br>
<a href="https://www.zybuluo.com/hanbingtao/note/476663" target="_blank">零基础入门深度学习(3) - 神经网络和反向传播算法</a> <br>
<a href="https://www.zybuluo.com/hanbingtao/note/485480" target="_blank">零基础入门深度学习(4) - 卷积神经网络</a> <br>
<a href="https://zybuluo.com/hanbingtao/note/541458" target="_blank">零基础入门深度学习(5) - 循环神经网络</a> <br>
<a href="https://zybuluo.com/hanbingtao/note/581764" target="_blank">零基础入门深度学习(6) - 长短时记忆网络(LSTM)</a> <br>
<a href="https://zybuluo.com/hanbingtao/note/626300" target="_blank">零基础入门深度学习(7) - 递归神经网络</a></p><div class="md-section-divider"></div><h2 data-anchor-id="bkiz" id="往期回顾">往期回顾</h2><p data-anchor-id="vhth">在前面的文章中，我们介绍了<strong>循环神经网络</strong>，它可以用来处理包含序列结构的信息。然而，除此之外，信息往往还存在着诸如树结构、图结构等更复杂的结构。对于这种复杂的结构，<strong>循环神经网络</strong>就无能为力了。本文介绍一种更为强大、复杂的神经网络：<strong>递归神经网络 (Recursive Neural Network, RNN)</strong>，以及它的训练算法<strong>BPTS (Back Propagation Through Structure)</strong>。顾名思义，<strong>递归神经网络</strong>（巧合的是，它的缩写和<strong>循环神经网络</strong>一样，也是RNN）可以处理诸如树、图这样的<strong>递归结构</strong>。在文章的最后，我们将实现一个<strong>递归神经网络</strong>，并介绍它的几个应用场景。</p><div class="md-section-divider"></div><h2 data-anchor-id="okoy" id="递归神经网络是啥">递归神经网络是啥</h2><p data-anchor-id="p92h">因为神经网络的输入层单元个数是固定的，因此必须用循环或者递归的方式来处理长度可变的输入。<strong>循环神经网络</strong>实现了前者，通过将长度不定的输入分割为等长度的小块，然后再依次的输入到网络中，从而实现了神经网络对变长输入的处理。一个典型的例子是，当我们处理一句话的时候，我们可以把一句话看作是词组成的序列，然后，每次向<strong>循环神经网络</strong>输入一个词，如此循环直至整句话输入完毕，<strong>循环神经网络</strong>将产生对应的输出。如此，我们就能处理任意长度的句子了。入下图所示：</p><p data-anchor-id="yzfv"><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-a87286a58fc03563.png" alt="" title=""></p><p data-anchor-id="p878">然而，有时候把句子看做是词的序列是不够的，比如下面这句话『两个外语学院的学生』：</p><p data-anchor-id="ok00"><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-1d69d9ea35f55af0.png" alt="" title=""></p><p data-anchor-id="c8xp">上图显示了这句话的两个不同的语法解析树。可以看出来这句话有歧义，不同的语法解析树则对应了不同的意思。一个是『两个外语学院的/学生』，也就是学生可能有许多，但他们来自于两所外语学校；另一个是『两个/外语学院的学生』，也就是只有两个学生，他们是外语学院的。为了能够让模型区分出两个不同的意思，我们的模型必须能够按照树结构去处理信息，而不是序列，这就是<strong>递归神经网络</strong>的作用。当面对按照树/图结构处理信息更有效的任务时，<strong>递归神经网络</strong>通常都会获得不错的结果。</p><p data-anchor-id="s576"><strong>递归神经网络</strong>可以把一个树/图结构信息编码为一个向量，也就是把信息映射到一个语义向量空间中。这个语义向量空间满足某类性质，比如语义相似的向量距离更近。也就是说，如果两句话（尽管内容不同）它的意思是相似的，那么把它们分别编码后的两个向量的距离也相近；反之，如果两句话的意思截然不同，那么编码后向量的距离则很远。如下图所示：</p><p data-anchor-id="1pik"><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-712ec37a73b854c1.png" alt=""></p><p data-anchor-id="dnii">从上图我们可以看到，<strong>递归神经网络</strong>将所有的词、句都映射到一个2维向量空间中。句子『the country of my birth』和句子『the place where I was born』的意思是非常接近的，所以表示它们的两个向量在向量空间中的距离很近。另外两个词『Germany』和『France』因为表示的都是地点，它们的向量与上面两句话的向量的距离，就比另外两个表示时间的词『Monday』和『Tuesday』的向量的距离近得多。这样，通过向量的距离，就得到了一种语义的表示。</p><p data-anchor-id="tjme">上图还显示了自然语言<strong>可组合</strong>的性质：词可以组成句、句可以组成段落、段落可以组成篇章，而更高层的语义取决于底层的语义以及它们的组合方式。<strong>递归神经网络</strong>是一种表示学习，它可以将词、句、段、篇按照他们的语义映射到同一个向量空间中，也就是把可组合（树/图结构）的信息表示为一个个有意义的向量。比如上面这个例子，<strong>递归神经网络</strong>把句子"the country of my birth"表示为二维向量[1,5]。有了这个『编码器』之后，我们就可以以这些有意义的向量为基础去完成更高级的任务（比如情感分析等）。如下图所示，<strong>递归神经网络</strong>在做情感分析时，可以比较好的处理否定句，这是胜过其他一些模型的：</p><p data-anchor-id="6soa"><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-1f91307ac606ba00.png" alt="" title=""></p><p data-anchor-id="43rm">在上图中，蓝色表示正面评价，红色表示负面评价。每个节点是一个向量，这个向量表达了以它为根的子树的情感评价。比如"intelligent humor"是正面评价，而"care about cleverness wit or any other kind of intelligent humor"是中性评价。我们可以看到，模型能够正确的处理doesn't的含义，将正面评价转变为负面评价。</p><p data-anchor-id="u57h">尽管<strong>递归神经网络</strong>具有更为强大的表示能力，但是在实际应用中并不太流行。其中一个主要原因是，<strong>递归神经网络</strong>的输入是树/图结构，而这种结构需要花费很多人工去标注。想象一下，如果我们用<strong>循环神经网络</strong>处理句子，那么我们可以直接把句子作为输入。然而，如果我们用<strong>递归神经网络</strong>处理句子，我们就必须把每个句子标注为语法解析树的形式，这无疑要花费非常大的精力。很多时候，相对于<strong>递归神经网络</strong>能够带来的性能提升，这个投入是不太划算的。</p><p data-anchor-id="wv0g">我们已经基本了解了<strong>递归神经网络</strong>是做什么用的，接下来，我们将探讨它的算法细节。</p><div class="md-section-divider"></div><h2 data-anchor-id="wz4s" id="递归神经网络的前向计算">递归神经网络的前向计算</h2><p data-anchor-id="0c9c">接下来，我们详细介绍一下<strong>递归神经网络</strong>是如何处理树/图结构的信息的。在这里，我们以处理树型信息为例进行介绍。</p><p data-anchor-id="bsww"><strong>递归神经网络</strong>的输入是两个子节点（也可以是多个），输出就是将这两个子节点编码后产生的父节点，父节点的维度和每个子节点是相同的。如下图所示：</p><p data-anchor-id="zuf1"><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-f2ea8885320110a5.png" alt="" title=""></p><p data-anchor-id="cqja"><span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-1-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -473.20006936799257 965.406943983867 643.1072455171717" style="width: 2.298ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-63"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="723" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-1">\mathbf{c}_1</script>和<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-2-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -473.20006936799257 965.406943983867 643.1072455171717" style="width: 2.298ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-63"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="723" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-2">\mathbf{c}_2</script>分别是表示两个子节点的向量，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-3-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -470.20006936799257 639.5 684.4001387359851" style="width: 1.452ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-70"></use></g></svg></span><script type="math/tex" id="MathJax-Element-3">\mathbf{p}</script>是表示父节点的向量。子节点和父节点组成一个全连接神经网络，也就是子节点的每个神经元都和父节点的每个神经元两两相连。我们用矩阵<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-4-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 1048.5 745.4001387359851" style="width: 2.419ex; height: 1.694ex; vertical-align: -0.242ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-57"></use></g></svg></span><script type="math/tex" id="MathJax-Element-4">W</script>表示这些连接上的权重，它的维度将是<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-5-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -714.2000693679926 2770.4444444444443 744.4001387359851" style="width: 6.411ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-64"></use><use href="#MJMAIN-D7" x="745" y="0"></use><use href="#MJMAIN-32" x="1746" y="0"></use><use href="#MJMATHI-64" x="2246" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-5">d\times 2d</script>，其中，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-6-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -714.2000693679926 523.5 744.4001387359851" style="width: 1.21ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-64"></use></g></svg></span><script type="math/tex" id="MathJax-Element-6">d</script>表示每个节点的维度。父节点的计算公式可以写成：</p><div class="md-section-divider"></div><p data-anchor-id="u8wm"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-7-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1469.7000693679925 14186.337214538518 2439.400138735985" style="width: 32.903ex; height: 5.685ex; vertical-align: -2.298ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-70"></use><use href="#MJMAIN-3D" x="917" y="0"></use><use href="#MJMATHI-74" x="1973" y="0"></use><use href="#MJMATHI-61" x="2335" y="0"></use><use href="#MJMATHI-6E" x="2864" y="0"></use><use href="#MJMATHI-68" x="3465" y="0"></use><use href="#MJMAIN-28" x="4041" y="0"></use><use href="#MJMATHI-57" x="4431" y="0"></use><g transform="translate(5479,0)"><use href="#MJSZ3-5B"></use><g transform="translate(695,0)"><g transform="translate(-15,0)"><g transform="translate(0,550)"><use href="#MJMAINB-63"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="723" y="-213"></use></g><g transform="translate(0,-650)"><use href="#MJMAINB-63"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="723" y="-213"></use></g></g></g><use href="#MJSZ3-5D" x="1813" y="-1"></use></g><use href="#MJMAIN-2B" x="8043" y="0"></use><use href="#MJMAINB-62" x="9044" y="0"></use><use href="#MJMAIN-29" x="9684" y="0"></use><use href="#MJMAIN-28" x="12073" y="0"></use><g transform="translate(12463,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-31" x="13296" y="0"></use><use href="#MJMAIN-29" x="13796" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-7">
\mathbf{p} = tanh(W\begin{bmatrix}\mathbf{c}_1\\\mathbf{c}_2\end{bmatrix}+\mathbf{b})\qquad(式1)
</script><p></p><p data-anchor-id="d04d">在上式中，tanh是激活函数（当然也可以用其它的激活函数），<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-8-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -714.2000693679926 639.5 740.4001387359851" style="width: 1.452ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-62"></use></g></svg></span><script type="math/tex" id="MathJax-Element-8">\mathbf{b}</script>是偏置项，它也是一个维度为<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-9-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -714.2000693679926 523.5 744.4001387359851" style="width: 1.21ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-64"></use></g></svg></span><script type="math/tex" id="MathJax-Element-9">d</script>的向量。如果读过前面的文章，相信大家已经非常熟悉这些计算了，在此不做过多的解释了。</p><p data-anchor-id="g0mn">然后，我们把产生的父节点的向量和其他子节点的向量再次作为网络的输入，再次产生它们的父节点。如此递归下去，直至整棵树处理完毕。最终，我们将得到根节点的向量，我们可以认为它是对整棵树的表示，这样我们就实现了把树映射为一个向量。在下图中，我们使用<strong>递归神经网络</strong>处理一棵树，最终得到的向量<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-10-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -470.20006936799257 1093.406943983867 736.0189337626031" style="width: 2.54ex; height: 1.694ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-70"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="904" y="-326"></use></g></svg></span><script type="math/tex" id="MathJax-Element-10">\mathbf{p}_3</script>，就是对整棵树的表示：</p><p data-anchor-id="d9f4"><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-2e300754026038f5.png" alt="" title=""></p><p data-anchor-id="ey9y">举个例子，我们使用<strong>递归神将网络</strong>将『两个外语学校的学生』映射为一个向量，如下图所示：</p><p data-anchor-id="l5vn"><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-577b3ccc7c34eb34.png" alt=""></p><p data-anchor-id="bk00">最后得到的向量<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-11-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -470.20006936799257 1093.406943983867 736.0189337626031" style="width: 2.54ex; height: 1.694ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-70"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="904" y="-326"></use></g></svg></span><script type="math/tex" id="MathJax-Element-11">\mathbf{p}_3</script>就是对整个句子『两个外语学校的学生』的表示。由于整个结构是递归的，不仅仅是根节点，事实上每个节点都是以其为根的子树的表示。比如，在左边的这棵树中，向量<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-12-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -470.20006936799257 1093.406943983867 720.462584576499" style="width: 2.54ex; height: 1.694ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-70"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="904" y="-326"></use></g></svg></span><script type="math/tex" id="MathJax-Element-12">\mathbf{p}_2</script>是短语『外语学院的学生』的表示，而向量<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-13-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -470.20006936799257 1093.406943983867 720.462584576499" style="width: 2.54ex; height: 1.694ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-70"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="904" y="-326"></use></g></svg></span><script type="math/tex" id="MathJax-Element-13">\mathbf{p}_1</script>是短语『外语学院的』的表示。</p><p data-anchor-id="6msr"><strong>式1</strong>就是<strong>递归神经网络</strong>的前向计算算法。它和全连接神经网络的计算没有什么区别，只是在输入的过程中需要根据输入的树结构依次输入每个子节点。</p><p data-anchor-id="if0a">需要特别注意的是，<strong>递归神经网络</strong>的权重<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-14-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 1048.5 745.4001387359851" style="width: 2.419ex; height: 1.694ex; vertical-align: -0.242ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-57"></use></g></svg></span><script type="math/tex" id="MathJax-Element-14">W</script>和偏置项<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-15-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -714.2000693679926 639.5 740.4001387359851" style="width: 1.452ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-62"></use></g></svg></span><script type="math/tex" id="MathJax-Element-15">\mathbf{b}</script>在所有的节点都是<strong>共享</strong>的。</p><div class="md-section-divider"></div><h2 data-anchor-id="caa6" id="递归神经网络的训练">递归神经网络的训练</h2><p data-anchor-id="aoz0"><strong>递归神经网络</strong>的训练算法和<strong>循环神经网络</strong>类似，两者不同之处在于，前者需要将残差<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-16-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -737.2000693679926 451.5 767.4001387359851" style="width: 1.089ex; height: 1.815ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use></g></svg></span><script type="math/tex" id="MathJax-Element-16">\delta</script>从根节点反向传播到各个子节点，而后者是将残差<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-17-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -737.2000693679926 451.5 767.4001387359851" style="width: 1.089ex; height: 1.815ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use></g></svg></span><script type="math/tex" id="MathJax-Element-17">\delta</script>从当前时刻<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-18-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -646.2000693679926 830.2561863887845 823.8854201102238" style="width: 1.935ex; height: 1.935ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="511" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-18">t_k</script>反向传播到初始时刻<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-19-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -646.2000693679926 815.406943983867 816.1072455171717" style="width: 1.935ex; height: 1.935ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="511" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-19">t_1</script>。 </p><p data-anchor-id="ea5o">下面，我们介绍适用于<strong>递归神经网络</strong>的训练算法，也就是<strong>BPTS</strong>算法。</p><div class="md-section-divider"></div><h3 data-anchor-id="ehg7" id="误差项的传递">误差项的传递</h3><p data-anchor-id="4pug">首先，我们先推导将误差从父节点传递到子节点的公式，如下图：</p><p data-anchor-id="y897"><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-9ab001431eb2f2a4.png" alt=""></p><p data-anchor-id="kblf">定义<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-20-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -737.2000693679926 900.5282643274268 1044.2859610673618" style="width: 2.056ex; height: 2.419ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="628" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-20">\delta_p</script>为误差函数E相对于父节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-21-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="-38.5 -463.20006936799257 542 677.4001387359851" style="width: 1.21ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-70"></use></g></svg></span><script type="math/tex" id="MathJax-Element-21">p</script>的加权输入<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-22-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -655.2000693679926 2070.5282643274268 962.2859610673619" style="width: 4.839ex; height: 2.177ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="2283" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-22">\mathbf{net}_p</script>的导数，即：</p><div class="md-section-divider"></div><p data-anchor-id="4ahi"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-23-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1411.7080693679925 5543.410080628286 2405.0018239705873" style="width: 12.823ex; height: 5.565ex; vertical-align: -2.419ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="628" y="-213"></use><g transform="translate(1178,0)"><use href="#MJMAIN-3D" x="155" y="0"></use><g transform="translate(0,624)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-64"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-65" x="523" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-66" x="990" y="0"></use></g></g><g transform="translate(2665,0)"><rect stroke="none" width="2758" height="60" x="0" y="220"></rect><g transform="translate(713,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="2283" y="-213"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-23">
\delta_p\overset{def}{=}\frac{\partial{E}}{\partial{\mathbf{net}_p}}
</script><p></p><p data-anchor-id="ecsx">设<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-24-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -655.2000693679926 2070.5282643274268 962.2859610673619" style="width: 4.839ex; height: 2.177ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="2283" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-24">\mathbf{net}_p</script>是父节点的<strong>加权输入</strong>，则</p><p data-anchor-id="p9pe"><span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-25-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1469.7000693679925 8657.5351562853 2439.400138735985" style="width: 20.081ex; height: 5.685ex; vertical-align: -2.298ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="2283" y="-213"></use><use href="#MJMAIN-3D" x="2348" y="0"></use><use href="#MJMATHI-57" x="3404" y="0"></use><g transform="translate(4453,0)"><use href="#MJSZ3-5B"></use><g transform="translate(695,0)"><g transform="translate(-15,0)"><g transform="translate(0,550)"><use href="#MJMAINB-63"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="723" y="-213"></use></g><g transform="translate(0,-650)"><use href="#MJMAINB-63"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="723" y="-213"></use></g></g></g><use href="#MJSZ3-5D" x="1813" y="-1"></use></g><use href="#MJMAIN-2B" x="7017" y="0"></use><use href="#MJMAINB-62" x="8018" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-25">\mathbf{net}_p=W\begin{bmatrix}\mathbf{c}_1\\\mathbf{c}_2\end{bmatrix}+\mathbf{b}</script></p><p data-anchor-id="wqhn">在上述式子里，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-26-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -655.2000693679926 2070.5282643274268 962.2859610673619" style="width: 4.839ex; height: 2.177ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="2283" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-26">\mathbf{net}_p</script>、<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-27-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -473.20006936799257 965.406943983867 643.1072455171717" style="width: 2.298ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-63"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="723" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-27">\mathbf{c}_1</script>、<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-28-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -473.20006936799257 965.406943983867 643.1072455171717" style="width: 2.298ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-63"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="723" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-28">\mathbf{c}_2</script>都是向量，而<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-29-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 1048.5 745.4001387359851" style="width: 2.419ex; height: 1.694ex; vertical-align: -0.242ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-57"></use></g></svg></span><script type="math/tex" id="MathJax-Element-29">W</script>是矩阵。为了看清楚它们的关系，我们将其展开：</p><div class="md-section-divider"></div><p data-anchor-id="u1pt"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-30-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -4969.200069367993 44270.673387096766 9438.400138735986" style="width: 102.823ex; height: 21.895ex; vertical-align: -10.524ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(3371,0)"><g transform="translate(-15,0)"><g transform="translate(0,2679)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-3106.0245061782125) scale(1,2.247138484540983)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-4215"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(25,1879)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-31" x="712" y="-326"></use></g></g></g><g transform="translate(25,539)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-32" x="712" y="-326"></use></g></g></g><g transform="translate(543,-661)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-1861)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6E" x="712" y="-326"></use></g></g></g></g></g><g transform="translate(3242,2679)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-3106.0245061782125) scale(1,2.247138484540983)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-4215"></use></g></g><g transform="translate(4173,0)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><g transform="translate(0,2609)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-2965.871110705) scale(1,2.014711626376451)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-4074"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(25,1809)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-31" x="712" y="-326"></use><g transform="translate(676,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><g transform="translate(306,-107)"><use transform="scale(0.5000000000000001)" href="#MJMAIN-31"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g></g><g transform="translate(25,609)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-32" x="712" y="-326"></use><g transform="translate(676,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><g transform="translate(306,-107)"><use transform="scale(0.5000000000000001)" href="#MJMAIN-31"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g></g><g transform="translate(626,-591)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-1791)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6E" x="712" y="-326"></use><g transform="translate(726,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><g transform="translate(306,-107)"><use transform="scale(0.5000000000000001)" href="#MJMAIN-31"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g></g></g><g transform="translate(3406,0)"><g transform="translate(25,1809)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-31" x="712" y="-326"></use><g transform="translate(676,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><g transform="translate(306,-107)"><use transform="scale(0.5000000000000001)" href="#MJMAIN-31"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g></g><g transform="translate(25,609)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-32" x="712" y="-326"></use><g transform="translate(676,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><g transform="translate(306,-107)"><use transform="scale(0.5000000000000001)" href="#MJMAIN-31"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g></g><g transform="translate(0,-1791)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6E" x="712" y="-326"></use><g transform="translate(726,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><g transform="translate(306,-107)"><use transform="scale(0.5000000000000001)" href="#MJMAIN-31"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g></g></g><g transform="translate(6828,0)"><g transform="translate(0,1809)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,609)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-1791)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(8996,0)"><g transform="translate(25,1809)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-31" x="712" y="-326"></use><g transform="translate(676,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><g transform="translate(306,-107)"><use transform="scale(0.5000000000000001)" href="#MJMAIN-31"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6E" x="500" y="0"></use></g></g></g></g><g transform="translate(25,609)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-32" x="712" y="-326"></use><g transform="translate(676,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><g transform="translate(306,-107)"><use transform="scale(0.5000000000000001)" href="#MJMAIN-31"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6E" x="500" y="0"></use></g></g></g></g><g transform="translate(0,-1791)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6E" x="712" y="-326"></use><g transform="translate(726,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><g transform="translate(306,-107)"><use transform="scale(0.5000000000000001)" href="#MJMAIN-31"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6E" x="500" y="0"></use></g></g></g></g></g><g transform="translate(12468,0)"><g transform="translate(25,1809)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-31" x="712" y="-326"></use><g transform="translate(676,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><g transform="translate(306,-107)"><use transform="scale(0.5000000000000001)" href="#MJMAIN-32"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g></g><g transform="translate(25,609)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-32" x="712" y="-326"></use><g transform="translate(676,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><g transform="translate(306,-107)"><use transform="scale(0.5000000000000001)" href="#MJMAIN-32"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g></g><g transform="translate(0,-1791)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6E" x="712" y="-326"></use><g transform="translate(726,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><g transform="translate(306,-107)"><use transform="scale(0.5000000000000001)" href="#MJMAIN-32"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g></g></g><g transform="translate(15889,0)"><g transform="translate(25,1809)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-31" x="712" y="-326"></use><g transform="translate(676,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><g transform="translate(306,-107)"><use transform="scale(0.5000000000000001)" href="#MJMAIN-32"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g></g><g transform="translate(25,609)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-32" x="712" y="-326"></use><g transform="translate(676,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><g transform="translate(306,-107)"><use transform="scale(0.5000000000000001)" href="#MJMAIN-32"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g></g><g transform="translate(0,-1791)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6E" x="712" y="-326"></use><g transform="translate(726,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><g transform="translate(306,-107)"><use transform="scale(0.5000000000000001)" href="#MJMAIN-32"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g></g></g><g transform="translate(19310,0)"><g transform="translate(0,1809)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,609)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-1791)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(21479,0)"><g transform="translate(25,1809)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-31" x="712" y="-326"></use><g transform="translate(676,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><g transform="translate(306,-107)"><use transform="scale(0.5000000000000001)" href="#MJMAIN-32"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6E" x="500" y="0"></use></g></g></g></g><g transform="translate(25,609)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-32" x="712" y="-326"></use><g transform="translate(676,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><g transform="translate(306,-107)"><use transform="scale(0.5000000000000001)" href="#MJMAIN-32"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6E" x="500" y="0"></use></g></g></g></g><g transform="translate(0,-1791)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6E" x="712" y="-326"></use><g transform="translate(726,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><g transform="translate(306,-107)"><use transform="scale(0.5000000000000001)" href="#MJMAIN-32"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6E" x="500" y="0"></use></g></g></g></g></g></g><g transform="translate(24952,2609)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-2965.871110705) scale(1,2.014711626376451)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-4074"></use></g></g><g transform="translate(26675,0)"><g transform="translate(0,4950)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-7639.235099337748) scale(1,9.764900662251655)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-8755"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(35,4150)"><use href="#MJMATHI-63"></use><g transform="translate(433,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g><g transform="translate(35,2950)"><use href="#MJMATHI-63"></use><g transform="translate(433,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><g transform="translate(71,1750)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,550)"><use href="#MJMATHI-63"></use><g transform="translate(433,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="500" y="0"></use></g></g><g transform="translate(35,-650)"><use href="#MJMATHI-63"></use><g transform="translate(433,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g><g transform="translate(35,-1850)"><use href="#MJMATHI-63"></use><g transform="translate(433,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><g transform="translate(71,-3050)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-4250)"><use href="#MJMATHI-63"></use><g transform="translate(433,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="500" y="0"></use></g></g></g></g><g transform="translate(2299,4950)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-7639.235099337748) scale(1,9.764900662251655)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-8755"></use></g></g><use href="#MJMAIN-2B" x="29864" y="0"></use><g transform="translate(30865,0)"><g transform="translate(0,2720)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-3187.3251413726084) scale(1,2.3819654085781234)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-4296"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(142,1625)"><use href="#MJMATHI-62"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="607" y="-213"></use></g><g transform="translate(142,379)"><use href="#MJMATHI-62"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="607" y="-213"></use></g><g transform="translate(0,-821)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(107,-2021)"><use href="#MJMATHI-62"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="607" y="-213"></use></g></g></g><g transform="translate(2155,2720)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-3187.3251413726084) scale(1,2.3819654085781234)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-4296"></use></g></g></g></g><g transform="translate(42191,0)"><g id="mjx-eqn-1"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-30">
\begin{align}
\begin{bmatrix}
net_{p_1}\\
net_{p_2}\\
...\\
net_{p_n}
\end{bmatrix}&=
\begin{bmatrix}
w_{p_1c_{11}}&w_{p_1c_{12}}&...&w_{p_1c_{1n}}&w_{p_1c_{21}}&w_{p_1c_{22}}&...&w_{p_1c_{2n}}\\
w_{p_2c_{11}}&w_{p_2c_{12}}&...&w_{p_2c_{1n}}&w_{p_2c_{21}}&w_{p_2c_{22}}&...&w_{p_2c_{2n}}\\
...\\
w_{p_nc_{11}}&w_{p_nc_{12}}&...&w_{p_nc_{1n}}&w_{p_nc_{21}}&w_{p_nc_{22}}&...&w_{p_nc_{2n}}\\
\end{bmatrix}
\begin{bmatrix}
c_{11}\\
c_{12}\\
...\\
c_{1n}\\
c_{21}\\
c_{22}\\
...\\
c_{2n}
\end{bmatrix}+\begin{bmatrix}\\
b_1\\
b_2\\
...\\
b_n\\
\end{bmatrix}
\end{align}
</script><p></p><p data-anchor-id="xmr8">在上面的公式中，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-31-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="-38.5 -463.20006936799257 886.3053928999522 677.4001387359851" style="width: 2.056ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-70"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="712" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-31">p_i</script>表示父节点p的第i个分量；<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-32-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -462.20006936799257 1131.7123368838193 639.8854201102238" style="width: 2.661ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-63"></use><g transform="translate(433,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="500" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-32">c_{1i}</script>表示<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-33-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -462.20006936799257 887.406943983867 632.1072455171717" style="width: 2.056ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-63"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="613" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-33">c_1</script>子节点的第i个分量；<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-34-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -462.20006936799257 1131.7123368838193 639.8854201102238" style="width: 2.661ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-63"></use><g transform="translate(433,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="500" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-34">c_{2i}</script>表示<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-35-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -462.20006936799257 887.406943983867 632.1072455171717" style="width: 2.056ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-63"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="613" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-35">c_2</script>子节点的第i个分量；<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-36-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 2260.230410209105 840.9661559139673" style="width: 5.202ex; height: 1.935ex; vertical-align: -0.968ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-69" x="712" y="-326"></use><g transform="translate(599,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><g transform="translate(306,-107)"><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6B" x="412" y="0"></use></g></g></g></g></svg></span><script type="math/tex" id="MathJax-Element-36">w_{p_ic_{jk}}</script>表示子节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-37-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -462.20006936799257 825.1815472394509 776.3570288792274" style="width: 1.935ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-63"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="613" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-37">c_j</script>的第k个分量到父节点p的第i个分量的的权重。根据上面展开后的矩阵乘法形式，我们不难看出，对于子节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-38-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -462.20006936799257 1193.9377336282355 776.3570288792274" style="width: 2.782ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-63"></use><g transform="translate(433,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="412" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-38">c_{jk}</script>来说，它会影响父节点所有的分量。因此，我们求误差函数E对<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-39-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -462.20006936799257 1193.9377336282355 776.3570288792274" style="width: 2.782ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-63"></use><g transform="translate(433,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="412" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-39">c_{jk}</script>的导数时，必须用到全导数公式，也就是：</p><div class="md-section-divider"></div><p data-anchor-id="0mvb"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-40-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -2841.502882505027 44270.673387096766 5183.005765010054" style="width: 102.823ex; height: 12.097ex; vertical-align: -5.565ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(16637,0)"><g transform="translate(-15,0)"><g transform="translate(0,1317)"><g transform="translate(120,0)"><rect stroke="none" width="1881" height="60" x="0" y="220"></rect><g transform="translate(274,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-63"></use><g transform="translate(433,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="412" y="0"></use></g></g></g></g></g></g><g transform="translate(2385,0)"><g transform="translate(0,1317)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJSZ2-2211"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="848" y="-1536"></use></g><g transform="translate(2667,0)"><g transform="translate(120,0)"><rect stroke="none" width="2815" height="60" x="0" y="220"></rect><g transform="translate(741,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-69" x="712" y="-326"></use></g></g></g></g></g></g><g transform="translate(6009,0)"><rect stroke="none" width="2815" height="60" x="0" y="220"></rect><g transform="translate(60,789)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><g transform="translate(361,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-69" x="712" y="-326"></use></g></g></g></g><g transform="translate(527,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-63"></use><g transform="translate(433,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="412" y="0"></use></g></g></g></g></g><g transform="translate(0,-1129)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJSZ2-2211"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="848" y="-1536"></use></g><g transform="translate(2667,0)"><use href="#MJMATHI-3B4"></use><g transform="translate(444,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-69" x="712" y="-326"></use></g></g><g transform="translate(3811,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-69" x="712" y="-326"></use><g transform="translate(599,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><g transform="translate(306,-107)"><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6B" x="412" y="0"></use></g></g></g></g></g></g></g><g transform="translate(42191,0)"><g transform="translate(0,1317)"><g id="mjx-eqn-2"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(0,-1129)"><g id="mjx-eqn-3"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-40">
\begin{align}
\frac{\partial{E}}{\partial{c_{jk}}}&=\sum_i{\frac{\partial{E}}{\partial{net_{p_i}}}}\frac{\partial{net_{p_i}}}{\partial{c_{jk}}}\\
&=\sum_i{\delta_{p_i}}w_{p_ic_{jk}}
\end{align}
</script><p></p><p data-anchor-id="vmtd">有了上式，我们就可以把它表示为矩阵形式，从而得到一个向量化表达：</p><div class="md-section-divider"></div><p data-anchor-id="eh65"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-41-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1456.0364458912265 44270.673387096766 2412.072891782453" style="width: 102.823ex; height: 5.565ex; vertical-align: -2.298ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(19739,0)"><g transform="translate(-15,0)"><g transform="translate(0,44)"><g transform="translate(120,0)"><rect stroke="none" width="1590" height="60" x="0" y="220"></rect><g transform="translate(129,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAINB-63"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="723" y="-213"></use></g></g></g></g></g><g transform="translate(2094,0)"><g transform="translate(0,44)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJMATHI-55"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="966" y="-213"></use></g><g transform="translate(2131,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="628" y="-213"></use></g></g></g></g><g transform="translate(42191,0)"><g transform="translate(0,44)"><g id="mjx-eqn-4"><use href="#MJMAIN-28"></use><use href="#MJMAIN-34" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-41">
\begin{align}
\frac{\partial{E}}{\partial{\mathbf{c}_j}}&=U_j\delta_p
\end{align}
</script><p></p><p data-anchor-id="ceyp">其中，矩阵<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-42-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 1075.181547239451 1017.3570288792274" style="width: 2.54ex; height: 2.419ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-55"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="966" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-42">U_j</script>是从矩阵W中提取部分元素组成的矩阵。其单元为：</p><div class="md-section-divider"></div><p data-anchor-id="yebw"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-135-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 5062.678191122766 840.9661559139673" style="width: 11.734ex; height: 1.935ex; vertical-align: -0.968ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-75"></use><g transform="translate(572,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><g transform="translate(291,-170)"><use transform="scale(0.5000000000000001)" href="#MJMATHI-69"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6B" x="345" y="0"></use></g></g><use href="#MJMAIN-3D" x="1746" y="0"></use><g transform="translate(2802,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6B" x="712" y="-326"></use><g transform="translate(687,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><g transform="translate(306,-107)"><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-69" x="412" y="0"></use></g></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-135">
u_{j_{ik}}=w_{p_kc_{ji}}
</script><p></p><p data-anchor-id="zz6h">上式看上去可能会让人晕菜，从下图，我们可以直观的看到<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-44-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 1075.181547239451 1017.3570288792274" style="width: 2.54ex; height: 2.419ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-55"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="966" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-44">U_j</script>到底是啥。首先我们把W矩阵拆分为两个矩阵<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-45-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 1398.406943983867 873.1072455171717" style="width: 3.266ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1335" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-45">W_1</script>和<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-46-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 1398.406943983867 873.1072455171717" style="width: 3.266ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="1335" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-46">W_2</script>，如下图所示：</p><p data-anchor-id="xh4f"><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-fb58d74fbc7e9ce5.png" alt=""></p><p data-anchor-id="e3bc">显然，子矩阵<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-47-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 1398.406943983867 873.1072455171717" style="width: 3.266ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1335" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-47">W_1</script>和<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-48-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 1398.406943983867 873.1072455171717" style="width: 3.266ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="1335" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-48">W_2</script>分别对应子节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-49-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -473.20006936799257 965.406943983867 643.1072455171717" style="width: 2.298ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-63"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="723" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-49">\mathbf{c}_1</script>和<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-50-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -473.20006936799257 965.406943983867 643.1072455171717" style="width: 2.298ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-63"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="723" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-50">\mathbf{c}_2</script>的到父节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-51-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -470.20006936799257 639.5 684.4001387359851" style="width: 1.452ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-70"></use></g></svg></span><script type="math/tex" id="MathJax-Element-51">\mathbf{p}</script>权重。则矩阵<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-52-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 1075.181547239451 1017.3570288792274" style="width: 2.54ex; height: 2.419ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-55"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="966" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-52">U_j</script>为：</p><div class="md-section-divider"></div><p data-anchor-id="bi8h"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-53-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -843.7687737936584 4087.1438301409294 1311.5872224503876" style="width: 9.435ex; height: 3.024ex; vertical-align: -1.21ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-55"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="966" y="-213"></use><use href="#MJMAIN-3D" x="1352" y="0"></use><g transform="translate(2409,0)"><use href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="1526" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="1335" y="-430"></use></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-53">
U_j=W_j^T
</script><p></p><p data-anchor-id="lnm7">也就是说，将误差项反向传递到相应子节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-54-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -473.20006936799257 903.1815472394509 787.3570288792274" style="width: 2.056ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-63"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="723" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-54">\mathbf{c}_j</script>的矩阵<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-55-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 1075.181547239451 1017.3570288792274" style="width: 2.54ex; height: 2.419ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-55"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="966" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-55">U_j</script>就是其对应权重矩阵<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-56-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 1336.181547239451 1017.3570288792274" style="width: 3.145ex; height: 2.419ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="1335" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-56">W_j</script>的转置。</p><p data-anchor-id="bh54">现在，我们设<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-57-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -655.2000693679926 2297.991467763023 1032.9661559139672" style="width: 5.323ex; height: 2.419ex; vertical-align: -0.968ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><g transform="translate(1614,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A" x="613" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-57">\mathbf{net}_{c_j}</script>是子节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-58-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -473.20006936799257 903.1815472394509 787.3570288792274" style="width: 2.056ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-63"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="723" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-58">\mathbf{c}_j</script>的加权输入，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-59-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -725.2000693679926 550.5 950.4001387359851" style="width: 1.331ex; height: 2.177ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-66"></use></g></svg></span><script type="math/tex" id="MathJax-Element-59">f</script>是子节点c的激活函数，则：</p><div class="md-section-divider"></div><p data-anchor-id="6h94"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-60-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -848.4830779569837 44270.673387096766 1196.9661559139674" style="width: 102.823ex; height: 2.782ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(19377,0)"><g transform="translate(-15,0)"><g transform="translate(0,29)"><use href="#MJMAINB-63"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="723" y="-213"></use><use href="#MJMAIN-3D" x="1180" y="0"></use><use href="#MJMATHI-66" x="2237" y="0"></use><use href="#MJMAIN-28" x="2787" y="0"></use><g transform="translate(3177,0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><g transform="translate(1614,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A" x="613" y="-213"></use></g></g><use href="#MJMAIN-29" x="5475" y="0"></use></g></g></g><g transform="translate(42191,0)"><g transform="translate(0,29)"><g id="mjx-eqn-5"><use href="#MJMAIN-28"></use><use href="#MJMAIN-35" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-60">
\begin{align}
\mathbf{c}_j=f(\mathbf{net}_{c_j})\\
\end{align}
</script><p></p><p data-anchor-id="tvez">这样，我们得到：</p><div class="md-section-divider"></div><p data-anchor-id="2bmb"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-61-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -3687.403867829636 44270.673387096766 6874.807735659272" style="width: 102.823ex; height: 15.968ex; vertical-align: -7.5ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(17346,0)"><g transform="translate(-15,0)"><g transform="translate(0,2275)"><use href="#MJMATHI-3B4"></use><g transform="translate(444,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A" x="613" y="-213"></use></g></g></g><g transform="translate(1391,0)"><g transform="translate(0,2275)"><use href="#MJMAIN-3D"></use><g transform="translate(1176,0)"><rect stroke="none" width="2985" height="60" x="0" y="220"></rect><g transform="translate(826,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><g transform="translate(1614,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A" x="613" y="-213"></use></g></g></g></g></g><g transform="translate(0,-551)"><use href="#MJMAIN-3D"></use><g transform="translate(1176,0)"><rect stroke="none" width="1590" height="60" x="0" y="220"></rect><g transform="translate(129,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAINB-63"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="723" y="-213"></use></g></g></g><g transform="translate(3173,0)"><rect stroke="none" width="2985" height="60" x="0" y="220"></rect><g transform="translate(757,765)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAINB-63"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="723" y="-213"></use></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><g transform="translate(1614,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A" x="613" y="-213"></use></g></g></g></g></g><g transform="translate(0,-2720)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="1526" y="488"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="1335" y="-430"></use></g><g transform="translate(2734,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="628" y="-213"></use></g><use href="#MJMAIN-2218" x="3856" y="0"></use><g transform="translate(4579,0)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use></g><use href="#MJMAIN-28" x="5443" y="0"></use><g transform="translate(5832,0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><g transform="translate(1614,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A" x="613" y="-213"></use></g></g><use href="#MJMAIN-29" x="8130" y="0"></use></g></g></g><g transform="translate(42191,0)"><g transform="translate(0,2275)"><g id="mjx-eqn-6"><use href="#MJMAIN-28"></use><use href="#MJMAIN-36" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(0,-551)"><g id="mjx-eqn-7"><use href="#MJMAIN-28"></use><use href="#MJMAIN-37" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(0,-2720)"><g id="mjx-eqn-8"><use href="#MJMAIN-28"></use><use href="#MJMAIN-38" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-61">
\begin{align}
\delta_{c_j}&=\frac{\partial{E}}{\partial{\mathbf{net}_{c_j}}}\\
&=\frac{\partial{E}}{\partial{\mathbf{c}_j}}\frac{\partial{\mathbf{c}_j}}{\partial{\mathbf{net}_{c_j}}}\\
&=W_j^T\delta_p\circ f'(\mathbf{net}_{c_j})
\end{align}
</script><p></p><p data-anchor-id="lrn1">如果我们将不同子节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-62-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -473.20006936799257 903.1815472394509 787.3570288792274" style="width: 2.056ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-63"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="723" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-62">\mathbf{c}_j</script>对应的误差项<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-63-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -737.2000693679926 1127.9914677630231 1114.9661559139674" style="width: 2.661ex; height: 2.54ex; vertical-align: -0.968ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><g transform="translate(444,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A" x="613" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-63">\delta_{c_j}</script>连接成一个向量<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-64-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1769.7000693679925 4843.677760936953 3039.400138735985" style="width: 11.25ex; height: 7.016ex; vertical-align: -3.024ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-63" x="628" y="-213"></use><use href="#MJMAIN-3D" x="1128" y="0"></use><g transform="translate(2185,0)"><use href="#MJSZ4-5B"></use><g transform="translate(750,0)"><g transform="translate(-15,0)"><g transform="translate(0,665)"><use href="#MJMATHI-3B4"></use><g transform="translate(444,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-31" x="613" y="-213"></use></g></g><g transform="translate(0,-709)"><use href="#MJMATHI-3B4"></use><g transform="translate(444,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-32" x="613" y="-213"></use></g></g></g></g><use href="#MJSZ4-5D" x="2075" y="-1"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-64">\delta_c=\begin{bmatrix}\delta_{c_1}\\\delta_{c_2}\end{bmatrix}</script>。那么，上式可以写成：</p><div class="md-section-divider"></div><p data-anchor-id="651r"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-65-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -911.5104670124718 13484.684811759626 1295.2925525094968" style="width: 31.331ex; height: 3.024ex; vertical-align: -0.968ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-63" x="628" y="-213"></use><use href="#MJMAIN-3D" x="1128" y="0"></use><g transform="translate(2185,0)"><use href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="1526" y="583"></use></g><g transform="translate(3862,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="628" y="-213"></use></g><use href="#MJMAIN-2218" x="4985" y="0"></use><g transform="translate(5708,0)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use></g><use href="#MJMAIN-28" x="6571" y="0"></use><g transform="translate(6961,0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-63" x="2283" y="-213"></use></g><use href="#MJMAIN-29" x="8982" y="0"></use><use href="#MJMAIN-28" x="11371" y="0"></use><g transform="translate(11761,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-32" x="12594" y="0"></use><use href="#MJMAIN-29" x="13095" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-65">
\delta_c=W^T\delta_p\circ f'(\mathbf{net}_c)\qquad(式2)
</script><p></p><p data-anchor-id="ih9x"><strong>式2</strong>就是将误差项从父节点传递到其子节点的公式。注意，上式中的<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-66-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -655.2000693679926 2021.0307896443683 833.5925268914102" style="width: 4.718ex; height: 1.935ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-63" x="2283" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-66">\mathbf{net}_c</script>也是将两个子节点的加权输入<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-67-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -655.2000693679926 2341.991467763023 930.9661559139672" style="width: 5.444ex; height: 2.177ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><g transform="translate(1614,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-31" x="613" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-67">\mathbf{net}_{c_1}</script>和<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-68-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -655.2000693679926 2341.991467763023 930.9661559139672" style="width: 5.444ex; height: 2.177ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><g transform="translate(1614,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-63"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-32" x="613" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-68">\mathbf{net}_{c_2}</script>连在一起的向量。</p><p data-anchor-id="puyw">有了传递一层的公式，我们就不难写出逐层传递的公式。</p><p data-anchor-id="tn63"><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-02c71c9fade90557.png" alt=""></p><p data-anchor-id="vucz">上图是在树型结构中反向传递误差项的全景图，反复应用<strong>式2</strong>，在已知<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-69-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1072.1642641169133 1458.3931265281876 1378.7623666631098" style="width: 3.387ex; height: 3.145ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="628" y="-212"></use></g></svg></span><script type="math/tex" id="MathJax-Element-69">\delta_p^{(3)}</script>的情况下，我们不难算出<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-70-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1072.1642641169133 1458.3931265281876 1378.7623666631098" style="width: 3.387ex; height: 3.145ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="628" y="-212"></use></g></svg></span><script type="math/tex" id="MathJax-Element-70">\delta_p^{(1)}</script>为：</p><div class="md-section-divider"></div><p data-anchor-id="lyjt"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-136-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -3400.412314375415 44270.673387096766 6300.82462875083" style="width: 102.823ex; height: 14.637ex; vertical-align: -6.774ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(16741,0)"><g transform="translate(-15,0)"><g transform="translate(0,2328)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(0,687)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="628" y="-212"></use></g><g transform="translate(0,-953)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(0,-2594)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="628" y="-212"></use></g></g><g transform="translate(1721,0)"><g transform="translate(0,2328)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="1526" y="583"></use></g><g transform="translate(2734,0)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="628" y="-212"></use></g><use href="#MJMAIN-2218" x="4414" y="0"></use><g transform="translate(5137,0)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use></g><use href="#MJMAIN-28" x="6000" y="0"></use><g transform="translate(6390,0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><g transform="translate(1614,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g></g><use href="#MJMAIN-29" x="9009" y="0"></use></g><g transform="translate(0,687)"><use href="#MJMAIN-3D"></use><use href="#MJMAIN-5B" x="1056" y="0"></use><g transform="translate(1334,0)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(2793,0)"><use href="#MJMAIN-5D"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="393" y="-213"></use></g></g><g transform="translate(0,-953)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="1526" y="583"></use></g><g transform="translate(2734,0)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="628" y="-212"></use></g><use href="#MJMAIN-2218" x="4414" y="0"></use><g transform="translate(5137,0)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use></g><use href="#MJMAIN-28" x="6000" y="0"></use><g transform="translate(6390,0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><g transform="translate(1614,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g></g><use href="#MJMAIN-29" x="9009" y="0"></use></g><g transform="translate(0,-2594)"><use href="#MJMAIN-3D"></use><use href="#MJMAIN-5B" x="1056" y="0"></use><g transform="translate(1334,0)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(2793,0)"><use href="#MJMAIN-5D"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="393" y="-213"></use></g></g></g></g><g transform="translate(41690,0)"><g transform="translate(500,2328)"><g id="mjx-eqn-9"><use href="#MJMAIN-28"></use><use href="#MJMAIN-39" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(0,687)"><g id="mjx-eqn-10"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-30" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-953)"><g id="mjx-eqn-11"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-31" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-2594)"><g id="mjx-eqn-12"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-32" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-136">
\begin{align}
\delta^{(2)}&=W^T\delta_p^{(3)}\circ f'(\mathbf{net}^{(2)})\\
\delta_p^{(2)}&=[\delta^{(2)}]_p\\
\delta^{(1)}&=W^T\delta_p^{(2)}\circ f'(\mathbf{net}^{(1)})\\
\delta_p^{(1)}&=[\delta^{(1)}]_p\\
\end{align}
</script><p></p><p data-anchor-id="7ylm">在上面的公式中，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-72-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1769.7000693679925 5737.441756585937 3039.400138735985" style="width: 13.306ex; height: 7.016ex; vertical-align: -3.024ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,362)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g><use href="#MJMAIN-3D" x="1736" y="0"></use><g transform="translate(2792,0)"><use href="#MJSZ4-5B"></use><g transform="translate(750,0)"><g transform="translate(-15,0)"><g transform="translate(0,672)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-63" x="628" y="-211"></use></g><g transform="translate(0,-939)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="628" y="-212"></use></g></g></g><use href="#MJSZ4-5D" x="2361" y="-1"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-72">\delta^{(2)}=\begin{bmatrix}\delta_c^{(2)}\\\delta_p^{(2)}\end{bmatrix}</script>，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-73-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -913.1292620390899 2471.4213908556144 1220.215153738459" style="width: 5.685ex; height: 2.782ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-5B"></use><g transform="translate(278,0)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,362)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(1736,0)"><use href="#MJMAIN-5D"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="393" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-73">[\delta^{(2)}]_p</script>表示取向量<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-74-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -913.1292620390899 1458.3931265281876 943.3293314070825" style="width: 3.387ex; height: 2.177ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,362)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="890" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-74">\delta^{(2)}</script>属于节点p的部分。</p><div class="md-section-divider"></div><h3 data-anchor-id="w6q5" id="权重梯度的计算">权重梯度的计算</h3><p data-anchor-id="5j3g">根据加权输入的计算公式：</p><div class="md-section-divider"></div><p data-anchor-id="olxf"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-75-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1072.1642641169133 8094.815113457011 1378.76236666311" style="width: 18.75ex; height: 3.145ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><g transform="translate(1614,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="2283" y="-212"></use><use href="#MJMAIN-3D" x="2754" y="0"></use><use href="#MJMATHI-57" x="3810" y="0"></use><g transform="translate(4858,0)"><use href="#MJMAINB-63"></use><g transform="translate(511,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g></g><use href="#MJMAIN-2B" x="6454" y="0"></use><use href="#MJMAINB-62" x="7455" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-75">
\mathbf{net}_p^{(l)}=W\mathbf{c}^{(l)}+\mathbf{b}
</script><p></p><p data-anchor-id="cjmn">其中，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-76-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1072.1642641169133 2476.407556728505 1378.76236666311" style="width: 5.806ex; height: 3.145ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><g transform="translate(1614,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="2283" y="-212"></use></g></svg></span><script type="math/tex" id="MathJax-Element-76">\mathbf{net}_p^{(l)}</script>表示第l层的父节点的加权输入，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-77-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -913.1292620390899 1373.407556728505 939.3293314070825" style="width: 3.145ex; height: 2.177ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-63"></use><g transform="translate(511,362)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-77">\mathbf{c}^{(l)}</script>表示第l层的子节点。<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-78-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 1048.5 745.4001387359851" style="width: 2.419ex; height: 1.694ex; vertical-align: -0.242ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-57"></use></g></svg></span><script type="math/tex" id="MathJax-Element-78">W</script>是权重矩阵，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-79-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -714.2000693679926 639.5 740.4001387359851" style="width: 1.452ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-62"></use></g></svg></span><script type="math/tex" id="MathJax-Element-79">\mathbf{b}</script>是偏置项。将其展开可得：</p><div class="md-section-divider"></div><p data-anchor-id="v65n"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-80-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -970.2000693679926 9467.129489391555 2183.656109255718" style="width: 22.016ex; height: 5.081ex; vertical-align: -2.903ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="2283" y="513"></use><g transform="translate(1614,-248)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A" x="712" y="-326"></use></g><use href="#MJMAIN-3D" x="2625" y="0"></use><g transform="translate(3681,0)"><use href="#MJSZ2-2211"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="848" y="-1536"></use></g><g transform="translate(5292,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g><g transform="translate(1352,0)"><use href="#MJMATHI-63"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="613" y="500"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="613" y="-430"></use></g></g><use href="#MJMAIN-2B" x="7645" y="0"></use><g transform="translate(8645,0)"><use href="#MJMATHI-62"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="607" y="-213"></use></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-80">
\mathbf{net}_{p_j}^l=\sum_i{w_{ji}c_i^l}+b_j
</script><p></p><p data-anchor-id="j5a5">那么，我们可以求得误差函数在第l层对权重的梯度为：</p><div class="md-section-divider"></div><p data-anchor-id="evzf"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-81-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -2858.061447217172 44270.673387096766 5216.122894434344" style="width: 102.823ex; height: 12.097ex; vertical-align: -5.565ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(16902,0)"><g transform="translate(-15,0)"><g transform="translate(0,900)"><g transform="translate(120,0)"><rect stroke="none" width="2265" height="60" x="0" y="220"></rect><g transform="translate(466,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-1024)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(716,-304)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></g></g></g></g><g transform="translate(2769,0)"><g transform="translate(0,900)"><use href="#MJMAIN-3D"></use><g transform="translate(1176,0)"><rect stroke="none" width="3163" height="60" x="0" y="220"></rect><g transform="translate(915,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-1024)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><g transform="translate(1614,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(1614,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A" x="712" y="-326"></use></g></g></g></g><g transform="translate(4746,0)"><rect stroke="none" width="3163" height="60" x="0" y="220"></rect><g transform="translate(60,885)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><g transform="translate(1614,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(1614,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A" x="712" y="-326"></use></g></g></g><g transform="translate(509,-1024)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(716,-304)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></g></g></g><g transform="translate(0,-1924)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(444,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A" x="712" y="-326"></use></g></g><g transform="translate(2371,0)"><use href="#MJMATHI-63"></use><g transform="translate(433,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="613" y="-430"></use></g></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,900)"><g id="mjx-eqn-13"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-33" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-1924)"><g id="mjx-eqn-14"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-34" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-81">
\begin{align}
\frac{\partial{E}}{\partial{w_{ji}^{(l)}}}&=\frac{\partial{E}}{\partial{\mathbf{net}_{p_j}^{(l)}}}\frac{\partial{\mathbf{net}_{p_j}^{(l)}}}{\partial{w_{ji}^{(l)}}}\\
&=\delta_{p_j}^{(l)}c_i^{(l)}\\
\end{align}
</script><p></p><p data-anchor-id="41tq">上式是针对一个权重项<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-82-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1352.486940139403 777.3570288792274" style="width: 3.145ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-82">w_{ji}</script>的公式，现在需要把它扩展为对所有的权重项的公式。我们可以把上式写成矩阵的形式（在下面的公式中，m=2n）：</p><div class="md-section-divider"></div><p data-anchor-id="c8pb"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-83-Frame" style="font-size: 100%; display: inline-block;"><span style="display: inline-block; white-space: nowrap; padding: 1px 0px;"><span style="display: inline-block; position: relative; width: 102.823ex; height: 51.411ex; vertical-align: -25.161ex;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -11336.59354218636 44270.673387096766 22173.187084372723" style="width: 102.823ex; height: 51.532ex; position: absolute; bottom: 0ex; left: 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(13378,0)"><g transform="translate(-15,0)"><g transform="translate(0,5109)"><g transform="translate(120,0)"><rect stroke="none" width="2629" height="60" x="0" y="220"></rect><g transform="translate(648,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-791)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-57"></use><g transform="translate(1079,288)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g></g></g></g></g></g><g transform="translate(3132,0)"><g transform="translate(0,5109)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><g transform="translate(0,6207)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-10150.158134103913) scale(1,13.928952129525559)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-11271"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(8,4815)"><g transform="translate(120,0)"><rect stroke="none" width="2265" height="60" x="0" y="220"></rect><g transform="translate(466,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-1024)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(716,-308)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g></g></g><g transform="translate(8,1683)"><g transform="translate(120,0)"><rect stroke="none" width="2265" height="60" x="0" y="220"></rect><g transform="translate(466,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-1024)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(716,-308)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g></g></g></g><g transform="translate(676,-1377)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-4369)"><g transform="translate(120,0)"><rect stroke="none" width="2282" height="60" x="0" y="220"></rect><g transform="translate(475,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-1024)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(716,-308)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="600" y="0"></use></g></g></g></g></g></g><g transform="translate(3508,0)"><g transform="translate(8,4815)"><g transform="translate(120,0)"><rect stroke="none" width="2265" height="60" x="0" y="220"></rect><g transform="translate(466,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-1024)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(716,-308)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g></g></g><g transform="translate(8,1683)"><g transform="translate(120,0)"><rect stroke="none" width="2265" height="60" x="0" y="220"></rect><g transform="translate(466,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-1024)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(716,-308)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g></g></g></g><g transform="translate(0,-4369)"><g transform="translate(120,0)"><rect stroke="none" width="2282" height="60" x="0" y="220"></rect><g transform="translate(475,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-1024)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(716,-308)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="600" y="0"></use></g></g></g></g></g></g><g transform="translate(7030,0)"><g transform="translate(0,4815)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,1683)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-4369)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(9199,0)"><g transform="translate(35,4815)"><g transform="translate(120,0)"><rect stroke="none" width="2479" height="60" x="0" y="220"></rect><g transform="translate(573,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-1024)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(716,-308)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-31"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6D" x="500" y="0"></use></g></g></g></g></g><g transform="translate(35,1683)"><g transform="translate(120,0)"><rect stroke="none" width="2479" height="60" x="0" y="220"></rect><g transform="translate(573,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-1024)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(716,-308)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-32"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6D" x="500" y="0"></use></g></g></g></g></g><g transform="translate(0,-4369)"><g transform="translate(120,0)"><rect stroke="none" width="2549" height="60" x="0" y="220"></rect><g transform="translate(608,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-1024)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6D" x="600" y="0"></use></g></g></g></g></g></g></g><g transform="translate(12990,6207)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-10150.158134103913) scale(1,13.928952129525559)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-11271"></use></g></g></g><g transform="translate(0,-5304)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><g transform="translate(0,4405)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-6552.226417737933) scale(1,7.962232865237036)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-7667"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,3352)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(444,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-31" x="712" y="-326"></use></g><g transform="translate(1315,0)"><use href="#MJMATHI-63"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="613" y="500"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="613" y="-435"></use></g></g><g transform="translate(0,1586)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(444,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-32" x="712" y="-326"></use></g><g transform="translate(1315,0)"><use href="#MJMATHI-63"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="613" y="500"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="613" y="-435"></use></g></g><g transform="translate(517,-935)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-3588)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(444,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6E" x="712" y="-326"></use></g><g transform="translate(1315,0)"><use href="#MJMATHI-63"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="613" y="500"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="613" y="-435"></use></g></g></g><g transform="translate(3188,0)"><g transform="translate(149,3352)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(444,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-31" x="712" y="-326"></use></g><g transform="translate(1315,0)"><use href="#MJMATHI-63"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="613" y="500"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="613" y="-435"></use></g></g><g transform="translate(149,1586)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(444,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-32" x="712" y="-326"></use></g><g transform="translate(1315,0)"><use href="#MJMATHI-63"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="613" y="500"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="613" y="-435"></use></g></g><g transform="translate(0,-3588)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(444,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6E" x="712" y="-326"></use></g><use href="#MJMATHI-6C" x="1315" y="0"></use><g transform="translate(1614,0)"><use href="#MJMATHI-63"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="613" y="500"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="613" y="-435"></use></g></g></g><g transform="translate(6690,0)"><g transform="translate(0,3352)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,1586)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-3588)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g></g><g transform="translate(8858,0)"><g transform="translate(25,3352)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="641" y="500"></use><g transform="translate(444,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-31" x="712" y="-326"></use></g><g transform="translate(1221,0)"><use href="#MJMATHI-63"></use><g transform="translate(433,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6D" x="613" y="-212"></use></g></g><g transform="translate(25,1586)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="641" y="500"></use><g transform="translate(444,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-32" x="712" y="-326"></use></g><g transform="translate(1221,0)"><use href="#MJMATHI-63"></use><g transform="translate(433,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6D" x="613" y="-212"></use></g></g><g transform="translate(0,-3588)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="641" y="500"></use><g transform="translate(444,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6E" x="712" y="-326"></use></g><g transform="translate(1271,0)"><use href="#MJMATHI-63"></use><g transform="translate(433,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6D" x="613" y="-212"></use></g></g></g></g><g transform="translate(12427,4405)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-6552.226417737933) scale(1,7.962232865237036)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-7667"></use></g></g></g><g transform="translate(0,-10453)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g></g><use href="#MJMAIN-28" x="2371" y="0"></use><g transform="translate(2761,0)"><use href="#MJMAINB-63"></use><g transform="translate(511,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g></g><g transform="translate(4134,0)"><use href="#MJMAIN-29"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="550" y="583"></use></g><use href="#MJMAIN-28" x="7122" y="0"></use><g transform="translate(7511,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-33" x="8345" y="0"></use><use href="#MJMAIN-29" x="8845" y="0"></use></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,5109)"><g id="mjx-eqn-15"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-35" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-5304)"><g id="mjx-eqn-16"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-36" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-10453)"><g id="mjx-eqn-17"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-37" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></span></span></div><script type="math/tex; mode=display" id="MathJax-Element-83">
\begin{align}
\frac{\partial{E}}{\partial{W^{(l)}}}&=
\begin{bmatrix}
\frac{\partial{E}}{\partial{w_{11}^{(l)}}}&
\frac{\partial{E}}{\partial{w_{12}^{(l)}}}&
...&
\frac{\partial{E}}{\partial{w_{1m}^{(l)}}}\\
\frac{\partial{E}}{\partial{w_{21}^{(l)}}}&
\frac{\partial{E}}{\partial{w_{22}^{(l)}}}&
...&
\frac{\partial{E}}{\partial{w_{2m}^{(l)}}}\\
\\\\\\...\\\\
\frac{\partial{E}}{\partial{w_{n1}^{(l)}}}&
\frac{\partial{E}}{\partial{w_{n2}^{(l)}}}&
...&
\frac{\partial{E}}{\partial{w_{nm}^{(l)}}}\\
\end{bmatrix}\\
&=
\begin{bmatrix}
\delta_{p_1}^{(l)}c_1^l&\delta_{p_1}^{(l)}c_2^l&...&\delta_{p_1}^lc_m^{(l)}\\
\delta_{p_2}^{(l)}c_1^l&\delta_{p_2}^{(l)}c_2^l&...&\delta_{p_2}^lc_m^{(l)}\\
\\\\\\...\\\\
\delta_{p_n}^{(l)}c_1^l&\delta_{p_n}^{(l)}lc_2^l&...&\delta_{p_n}^lc_m^{(l)}\\
\end{bmatrix}\\
&=\delta^{{(l)}}(\mathbf{c}^{(l)})^T\qquad(式3)
\end{align}
</script><p></p><p data-anchor-id="7edp"><strong>式3</strong>就是第l层权重项的梯度计算公式。我们知道，由于权重<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-84-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 1048.5 745.4001387359851" style="width: 2.419ex; height: 1.694ex; vertical-align: -0.242ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-57"></use></g></svg></span><script type="math/tex" id="MathJax-Element-84">W</script>是在所有层共享的，所以和<strong>循环神经网络</strong>一样，<strong>递归神经网络</strong>的最终的<strong>权重梯度是各个层权重梯度之和</strong>。即：</p><div class="md-section-divider"></div><p data-anchor-id="bb75"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-85-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1411.7080693679925 12069.876768198039 2649.912846597247" style="width: 28.065ex; height: 6.169ex; vertical-align: -3.024ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(120,0)"><rect stroke="none" width="1736" height="60" x="0" y="220"></rect><g transform="translate(202,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g><use href="#MJMAIN-3D" x="2253" y="0"></use><g transform="translate(3310,0)"><use href="#MJSZ2-2211"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="872" y="-1570"></use></g><g transform="translate(5041,0)"><rect stroke="none" width="2629" height="60" x="0" y="220"></rect><g transform="translate(648,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-791)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-57"></use><g transform="translate(1079,288)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g></g></g></g><use href="#MJMAIN-28" x="9957" y="0"></use><g transform="translate(10346,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-34" x="11179" y="0"></use><use href="#MJMAIN-29" x="11680" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-85">
\frac{\partial{E}}{\partial{W}}=\sum_l\frac{\partial{E}}{\partial{W^{(l)}}}\qquad(式4)
</script><p></p><p data-anchor-id="0gyj">因为<strong>循环神经网络</strong>的证明过程已经在<a href="https://www.zybuluo.com/hanbingtao/note/485480" target="_blank">零基础入门深度学习(4) - 卷积神经网络</a>一文中给出，因此，<strong>递归神经网络</strong>『为什么最终梯度是各层梯度之和』的证明就留给读者自行完成啦。</p><p data-anchor-id="i2up">接下来，我们求偏置项<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-86-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -714.2000693679926 639.5 740.4001387359851" style="width: 1.452ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-62"></use></g></svg></span><script type="math/tex" id="MathJax-Element-86">\mathbf{b}</script>的梯度计算公式。先计算误差函数对第l层偏置项<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-87-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -972.2176769687143 1501.407556728505 998.4177463367068" style="width: 3.508ex; height: 2.298ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-62"></use><g transform="translate(639,421)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-87">\mathbf{b}^{(l)}</script>的梯度：</p><div class="md-section-divider"></div><p data-anchor-id="iy1b"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-88-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -2858.061447217172 44270.673387096766 5216.122894434344" style="width: 102.823ex; height: 12.097ex; vertical-align: -5.565ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(17045,0)"><g transform="translate(-15,0)"><g transform="translate(0,900)"><g transform="translate(120,0)"><rect stroke="none" width="1978" height="60" x="0" y="220"></rect><g transform="translate(323,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-1024)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-62"></use><g transform="translate(429,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="607" y="-430"></use></g></g></g></g></g><g transform="translate(2482,0)"><g transform="translate(0,900)"><use href="#MJMAIN-3D"></use><g transform="translate(1176,0)"><rect stroke="none" width="3163" height="60" x="0" y="220"></rect><g transform="translate(915,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-1024)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><g transform="translate(1614,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(1614,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A" x="712" y="-326"></use></g></g></g></g><g transform="translate(4746,0)"><rect stroke="none" width="3163" height="60" x="0" y="220"></rect><g transform="translate(60,885)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAINB-6E"></use><use href="#MJMAINB-65" x="639" y="0"></use><use href="#MJMAINB-74" x="1167" y="0"></use><g transform="translate(1614,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(1614,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A" x="712" y="-326"></use></g></g></g><g transform="translate(652,-1024)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-62"></use><g transform="translate(429,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="607" y="-430"></use></g></g></g></g><g transform="translate(0,-1924)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(444,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A" x="712" y="-326"></use></g></g></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,900)"><g id="mjx-eqn-18"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-38" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-1924)"><g id="mjx-eqn-19"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-39" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-88">
\begin{align}
\frac{\partial{E}}{\partial{b_j^{(l)}}}&=\frac{\partial{E}}{\partial{\mathbf{net}_{p_j}^{(l)}}}\frac{\partial{\mathbf{net}_{p_j}^{(l)}}}{\partial{b_j^{(l)}}}\\
&=\delta_{p_j}^{(l)}\\
\end{align}
</script><p></p><p data-anchor-id="oq9n">把上式扩展为矩阵的形式：</p><div class="md-section-divider"></div><p data-anchor-id="dz7v"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-89-Frame" style="font-size: 100%; display: inline-block;"><span style="display: inline-block; white-space: nowrap; padding: 1px 0px;"><span style="display: inline-block; position: relative; width: 102.823ex; height: 51.29ex; vertical-align: -25.04ex;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -11304.49046252992 44270.673387096766 22108.980925059845" style="width: 102.823ex; height: 51.29ex; position: absolute; bottom: 0ex; left: 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(17713,0)"><g transform="translate(-15,0)"><g transform="translate(0,5164)"><g transform="translate(120,0)"><rect stroke="none" width="2188" height="60" x="0" y="220"></rect><g transform="translate(428,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-924)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAINB-62"></use><g transform="translate(639,421)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g></g></g></g></g></g><g transform="translate(2692,0)"><g transform="translate(0,5164)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><g transform="translate(0,6120)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-9977.203795828094) scale(1,13.642129014640288)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-11097"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,4728)"><g transform="translate(120,0)"><rect stroke="none" width="1978" height="60" x="0" y="220"></rect><g transform="translate(323,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-1024)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-62"></use><g transform="translate(429,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="607" y="-435"></use></g></g></g></g><g transform="translate(0,1604)"><g transform="translate(120,0)"><rect stroke="none" width="1978" height="60" x="0" y="220"></rect><g transform="translate(323,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-1024)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-62"></use><g transform="translate(429,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="607" y="-435"></use></g></g></g></g><g transform="translate(525,-1448)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-4440)"><g transform="translate(120,0)"><rect stroke="none" width="1978" height="60" x="0" y="220"></rect><g transform="translate(323,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-1024)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-62"></use><g transform="translate(429,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="607" y="-212"></use></g></g></g></g></g></g><g transform="translate(3206,6120)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-9977.203795828094) scale(1,13.642129014640288)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-11097"></use></g></g></g><g transform="translate(0,-5163)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><g transform="translate(0,4405)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-6552.226417737933) scale(1,7.962232865237036)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-7667"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,3352)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(444,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-31" x="712" y="-326"></use></g></g><g transform="translate(0,1586)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(444,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMAIN-32" x="712" y="-326"></use></g></g><g transform="translate(73,-935)"><use href="#MJMAIN-2E"></use><use href="#MJMAIN-2E" x="445" y="0"></use><use href="#MJMAIN-2E" x="890" y="0"></use></g><g transform="translate(0,-3588)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><g transform="translate(444,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-70"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6E" x="712" y="-326"></use></g></g></g></g><g transform="translate(2302,4405)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-6552.226417737933) scale(1,7.962232865237036)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-7667"></use></g></g></g><g transform="translate(0,-10421)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,521)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="628" y="-212"></use></g><use href="#MJMAIN-28" x="4371" y="0"></use><g transform="translate(4761,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-35" x="5594" y="0"></use><use href="#MJMAIN-29" x="6095" y="0"></use></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,5164)"><g id="mjx-eqn-20"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-30" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-5163)"><g id="mjx-eqn-21"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-31" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-10421)"><g id="mjx-eqn-22"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-32" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></span></span></div><script type="math/tex; mode=display" id="MathJax-Element-89">
\begin{align}
\frac{\partial{E}}{\partial{\mathbf{b}^{(l)}}}&=
\begin{bmatrix}
\frac{\partial{E}}{\partial{b_1^{(l)}}}\\
\frac{\partial{E}}{\partial{b_2^{(l)}}}\\
\\\\\\...\\\\
\frac{\partial{E}}{\partial{b_n^{(l)}}}\\
\end{bmatrix}\\
&=
\begin{bmatrix}
\delta_{p_1}^{(l)}\\
\delta_{p_2}^{(l)}\\
\\\\\\...\\\\
\delta_{p_n}^{(l)}\\
\end{bmatrix}\\
&=\delta_p^{{(l)}}\qquad(式5)
\end{align}
</script><p></p><p data-anchor-id="xtig"><strong>式5</strong>是第l层偏置项的梯度，那么最终的偏置项梯度是各个层偏置项梯度之和，即：</p><div class="md-section-divider"></div><p data-anchor-id="yrma"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-90-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1411.7080693679925 11345.626768198039 2649.912846597247" style="width: 26.371ex; height: 6.169ex; vertical-align: -3.024ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(120,0)"><rect stroke="none" width="1452" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(122,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMAINB-62" x="567" y="0"></use></g></g><use href="#MJMAIN-3D" x="1969" y="0"></use><g transform="translate(3026,0)"><use href="#MJSZ2-2211"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="872" y="-1570"></use></g><g transform="translate(4757,0)"><rect stroke="none" width="2188" height="60" x="0" y="220"></rect><g transform="translate(428,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-924)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMAINB-62"></use><g transform="translate(639,421)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g></g></g></g><use href="#MJMAIN-28" x="9232" y="0"></use><g transform="translate(9622,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-36" x="10455" y="0"></use><use href="#MJMAIN-29" x="10956" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-90">
\frac{\partial{E}}{\partial{\mathbf{b}}}=\sum_l\frac{\partial{E}}{\partial{\mathbf{b}^{(l)}}}\qquad(式6)
</script><p></p><div class="md-section-divider"></div><h3 data-anchor-id="1k2v" id="权重更新">权重更新</h3><p data-anchor-id="r11h">如果使用梯度下降优化算法，那么权重更新公式为：</p><div class="md-section-divider"></div><p data-anchor-id="ygag"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-91-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1411.7080693679925 7522.166666666667 2140.1160016392105" style="width: 17.419ex; height: 4.96ex; vertical-align: -1.815ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-57"></use><use href="#MJMAIN-2190" x="1326" y="0"></use><use href="#MJMATHI-57" x="2604" y="0"></use><use href="#MJMAIN-2B" x="3875" y="0"></use><use href="#MJMATHI-3B7" x="4876" y="0"></use><g transform="translate(5666,0)"><rect stroke="none" width="1736" height="60" x="0" y="220"></rect><g transform="translate(202,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-57" x="567" y="0"></use></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-91">
W\gets W + \eta\frac{\partial{E}}{\partial{W}}
</script><p></p><p data-anchor-id="nwgf">其中，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-92-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 503.5 699.4001387359851" style="width: 1.21ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B7"></use></g></svg></span><script type="math/tex" id="MathJax-Element-92">\eta</script>是学习速率常数。把<strong>式4</strong>带入到上式，即可完成权重的更新。同理，偏置项的更新公式为：</p><div class="md-section-divider"></div><p data-anchor-id="kdxx"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-93-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1411.7080693679925 6420.166666666667 2140.1160016392105" style="width: 14.879ex; height: 4.96ex; vertical-align: -1.815ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-62"></use><use href="#MJMAIN-2190" x="917" y="0"></use><use href="#MJMAINB-62" x="2195" y="0"></use><use href="#MJMAIN-2B" x="3057" y="0"></use><use href="#MJMATHI-3B7" x="4058" y="0"></use><g transform="translate(4848,0)"><rect stroke="none" width="1452" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-45" x="567" y="0"></use></g><g transform="translate(122,-687)"><use href="#MJMAIN-2202"></use><use href="#MJMAINB-62" x="567" y="0"></use></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-93">
\mathbf{b}\gets \mathbf{b} + \eta\frac{\partial{E}}{\partial{\mathbf{b}}}
</script><p></p><p data-anchor-id="eq9h">把<strong>式6</strong>带入到上式，即可完成偏置项的更新。</p><p data-anchor-id="0i0x">这就是<strong>递归神经网络</strong>的训练算法BPTS。由于我们有了前面几篇文章的基础，相信读者们理解BPTS算法也会比较容易。</p><div class="md-section-divider"></div><h2 data-anchor-id="8mds" id="递归神经网络的实现">递归神经网络的实现</h2><blockquote data-anchor-id="ssy7" class="white-blockquote">
  <p>完整代码请参考GitHub: <a href="https://github.com/hanbt/learn_dl/blob/master/recursive.py" target="_blank">https://github.com/hanbt/learn_dl/blob/master/recursive.py</a> (python2.7)</p>
</blockquote><p data-anchor-id="qagq">现在，我们实现一个处理树型结构的<strong>递归神经网络</strong>。</p><p data-anchor-id="4z3o">在文件的开头，加入如下代码：</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="5ydd" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="com">#!/usr/bin/env python</span></code></li><li class="L1"><code class="language-python"><span class="com"># -*- coding: UTF-8 -*-</span></code></li><li class="L2"><code class="language-python"></code></li><li class="L3"><code class="language-python"></code></li><li class="L4"><code class="language-python"><span class="kwd">import</span><span class="pln"> numpy </span><span class="kwd">as</span><span class="pln"> np</span></code></li><li class="L5"><code class="language-python"><span class="kwd">from</span><span class="pln"> cnn </span><span class="kwd">import</span><span class="pln"> </span><span class="typ">IdentityActivator</span></code></li></ol></pre><p data-anchor-id="7tde">上述四行代码非常简单，没有什么需要解释的。IdentityActivator激活函数是在我们介绍<strong>卷积神经网络</strong>时写的，现在引用一下它。</p><p data-anchor-id="ns3n">我们首先定义一个树节点结构，这样，我们就可以用它保存卷积神经网络生成的整棵树：</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="r2ls" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="kwd">class</span><span class="pln"> </span><span class="typ">TreeNode</span><span class="pun">(</span><span class="pln">object</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> __init__</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> data</span><span class="pun">,</span><span class="pln"> children</span><span class="pun">=[],</span><span class="pln"> children_data</span><span class="pun">=[]):</span></code></li><li class="L2"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">parent </span><span class="pun">=</span><span class="pln"> </span><span class="kwd">None</span></code></li><li class="L3"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">children </span><span class="pun">=</span><span class="pln"> children</span></code></li><li class="L4"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">children_data </span><span class="pun">=</span><span class="pln"> children_data</span></code></li><li class="L5"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">data </span><span class="pun">=</span><span class="pln"> data</span></code></li><li class="L6"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> child </span><span class="kwd">in</span><span class="pln"> children</span><span class="pun">:</span></code></li><li class="L7"><code class="language-python"><span class="pln">            child</span><span class="pun">.</span><span class="pln">parent </span><span class="pun">=</span><span class="pln"> self</span></code></li></ol></pre><p data-anchor-id="l31e">接下来，我们把<strong>递归神经网络</strong>的实现代码都放在RecursiveLayer类中，下面是这个类的构造函数：</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="lbx1" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="com"># 递归神经网络实现</span></code></li><li class="L1"><code class="language-python"><span class="kwd">class</span><span class="pln"> </span><span class="typ">RecursiveLayer</span><span class="pun">(</span><span class="pln">object</span><span class="pun">):</span></code></li><li class="L2"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> __init__</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> node_width</span><span class="pun">,</span><span class="pln"> child_count</span><span class="pun">,</span><span class="pln"> </span></code></li><li class="L3"><code class="language-python"><span class="pln">                 activator</span><span class="pun">,</span><span class="pln"> learning_rate</span><span class="pun">):</span></code></li><li class="L4"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L5"><code class="language-python"><span class="str">        递归神经网络构造函数</span></code></li><li class="L6"><code class="language-python"><span class="str">        node_width: 表示每个节点的向量的维度</span></code></li><li class="L7"><code class="language-python"><span class="str">        child_count: 每个父节点有几个子节点</span></code></li><li class="L8"><code class="language-python"><span class="str">        activator: 激活函数对象</span></code></li><li class="L9"><code class="language-python"><span class="str">        learning_rate: 梯度下降算法学习率</span></code></li><li class="L0"><code class="language-python"><span class="str">        '''</span></code></li><li class="L1"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">node_width </span><span class="pun">=</span><span class="pln"> node_width</span></code></li><li class="L2"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">child_count </span><span class="pun">=</span><span class="pln"> child_count</span></code></li><li class="L3"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">activator </span><span class="pun">=</span><span class="pln"> activator</span></code></li><li class="L4"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">learning_rate </span><span class="pun">=</span><span class="pln"> learning_rate</span></code></li><li class="L5"><code class="language-python"><span class="pln">        </span><span class="com"># 权重数组W</span></code></li><li class="L6"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">W </span><span class="pun">=</span><span class="pln"> np</span><span class="pun">.</span><span class="pln">random</span><span class="pun">.</span><span class="pln">uniform</span><span class="pun">(-</span><span class="lit">1e-4</span><span class="pun">,</span><span class="pln"> </span><span class="lit">1e-4</span><span class="pun">,</span></code></li><li class="L7"><code class="language-python"><span class="pln">            </span><span class="pun">(</span><span class="pln">node_width</span><span class="pun">,</span><span class="pln"> node_width </span><span class="pun">*</span><span class="pln"> child_count</span><span class="pun">))</span></code></li><li class="L8"><code class="language-python"><span class="pln">        </span><span class="com"># 偏置项b</span></code></li><li class="L9"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">b </span><span class="pun">=</span><span class="pln"> np</span><span class="pun">.</span><span class="pln">zeros</span><span class="pun">((</span><span class="pln">node_width</span><span class="pun">,</span><span class="pln"> </span><span class="lit">1</span><span class="pun">))</span></code></li><li class="L0"><code class="language-python"><span class="pln">        </span><span class="com"># 递归神经网络生成的树的根节点</span></code></li><li class="L1"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">root </span><span class="pun">=</span><span class="pln"> </span><span class="kwd">None</span></code></li></ol></pre><p data-anchor-id="a2ps">下面是前向计算的实现：</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="1vp2" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> forward</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> </span><span class="pun">*</span><span class="pln">children</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L2"><code class="language-python"><span class="str">        前向计算</span></code></li><li class="L3"><code class="language-python"><span class="str">        '''</span></code></li><li class="L4"><code class="language-python"><span class="pln">        children_data </span><span class="pun">=</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">concatenate</span><span class="pun">(</span><span class="pln">children</span><span class="pun">)</span></code></li><li class="L5"><code class="language-python"><span class="pln">        parent_data </span><span class="pun">=</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">activator</span><span class="pun">.</span><span class="pln">forward</span><span class="pun">(</span></code></li><li class="L6"><code class="language-python"><span class="pln">            np</span><span class="pun">.</span><span class="pln">dot</span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">W</span><span class="pun">,</span><span class="pln"> children_data</span><span class="pun">)</span><span class="pln"> </span><span class="pun">+</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">b</span></code></li><li class="L7"><code class="language-python"><span class="pln">        </span><span class="pun">)</span></code></li><li class="L8"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">root </span><span class="pun">=</span><span class="pln"> </span><span class="typ">TreeNode</span><span class="pun">(</span><span class="pln">parent_data</span><span class="pun">,</span><span class="pln"> children</span></code></li><li class="L9"><code class="language-python"><span class="pln">                            </span><span class="pun">,</span><span class="pln"> children_data</span><span class="pun">)</span></code></li></ol></pre><p data-anchor-id="nwkl">forward函数接收一系列的树节点对象作为输入，然后，<strong>递归神经网络</strong>将这些树节点作为子节点，并计算它们的父节点。最后，将计算的父节点保存在self.root变量中。</p><p data-anchor-id="dvsw">上面用到的concatenate函数，是将各个子节点中的数据拼接成一个长向量，其代码如下：</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="wthk" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> concatenate</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> tree_nodes</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L2"><code class="language-python"><span class="str">        将各个树节点中的数据拼接成一个长向量</span></code></li><li class="L3"><code class="language-python"><span class="str">        '''</span></code></li><li class="L4"><code class="language-python"><span class="pln">        concat </span><span class="pun">=</span><span class="pln"> np</span><span class="pun">.</span><span class="pln">zeros</span><span class="pun">((</span><span class="lit">0</span><span class="pun">,</span><span class="lit">1</span><span class="pun">))</span></code></li><li class="L5"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> node </span><span class="kwd">in</span><span class="pln"> tree_nodes</span><span class="pun">:</span></code></li><li class="L6"><code class="language-python"><span class="pln">            concat </span><span class="pun">=</span><span class="pln"> np</span><span class="pun">.</span><span class="pln">concatenate</span><span class="pun">((</span><span class="pln">concat</span><span class="pun">,</span><span class="pln"> node</span><span class="pun">.</span><span class="pln">data</span><span class="pun">))</span></code></li><li class="L7"><code class="language-python"><span class="pln">        </span><span class="kwd">return</span><span class="pln"> concat</span></code></li></ol></pre><p data-anchor-id="vioz">下面是反向传播算法BPTS的实现：</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="fnql" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> backward</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> parent_delta</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L2"><code class="language-python"><span class="str">        BPTS反向传播算法</span></code></li><li class="L3"><code class="language-python"><span class="str">        '''</span></code></li><li class="L4"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">calc_delta</span><span class="pun">(</span><span class="pln">parent_delta</span><span class="pun">,</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">root</span><span class="pun">)</span></code></li><li class="L5"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">W_grad</span><span class="pun">,</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">b_grad </span><span class="pun">=</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">calc_gradient</span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">root</span><span class="pun">)</span></code></li><li class="L6"><code class="language-python"></code></li><li class="L7"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> calc_delta</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> parent_delta</span><span class="pun">,</span><span class="pln"> parent</span><span class="pun">):</span></code></li><li class="L8"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L9"><code class="language-python"><span class="str">        计算每个节点的delta</span></code></li><li class="L0"><code class="language-python"><span class="str">        '''</span></code></li><li class="L1"><code class="language-python"><span class="pln">        parent</span><span class="pun">.</span><span class="pln">delta </span><span class="pun">=</span><span class="pln"> parent_delta</span></code></li><li class="L2"><code class="language-python"><span class="pln">        </span><span class="kwd">if</span><span class="pln"> parent</span><span class="pun">.</span><span class="pln">children</span><span class="pun">:</span></code></li><li class="L3"><code class="language-python"><span class="pln">            </span><span class="com"># 根据式2计算每个子节点的delta</span></code></li><li class="L4"><code class="language-python"><span class="pln">            children_delta </span><span class="pun">=</span><span class="pln"> np</span><span class="pun">.</span><span class="pln">dot</span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">W</span><span class="pun">.</span><span class="pln">T</span><span class="pun">,</span><span class="pln"> parent_delta</span><span class="pun">)</span><span class="pln"> </span><span class="pun">*</span><span class="pln"> </span><span class="pun">(</span></code></li><li class="L5"><code class="language-python"><span class="pln">                self</span><span class="pun">.</span><span class="pln">activator</span><span class="pun">.</span><span class="pln">backward</span><span class="pun">(</span><span class="pln">parent</span><span class="pun">.</span><span class="pln">children_data</span><span class="pun">)</span></code></li><li class="L6"><code class="language-python"><span class="pln">            </span><span class="pun">)</span></code></li><li class="L7"><code class="language-python"><span class="pln">            </span><span class="com"># slices = [(子节点编号，子节点delta起始位置，子节点delta结束位置)]</span></code></li><li class="L8"><code class="language-python"><span class="pln">            slices </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[(</span><span class="pln">i</span><span class="pun">,</span><span class="pln"> i </span><span class="pun">*</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">node_width</span><span class="pun">,</span><span class="pln"> </span></code></li><li class="L9"><code class="language-python"><span class="pln">                        </span><span class="pun">(</span><span class="pln">i </span><span class="pun">+</span><span class="pln"> </span><span class="lit">1</span><span class="pun">)</span><span class="pln"> </span><span class="pun">*</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">node_width</span><span class="pun">)</span></code></li><li class="L0"><code class="language-python"><span class="pln">                        </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">child_count</span><span class="pun">)]</span></code></li><li class="L1"><code class="language-python"><span class="pln">            </span><span class="com"># 针对每个子节点，递归调用calc_delta函数</span></code></li><li class="L2"><code class="language-python"><span class="pln">            </span><span class="kwd">for</span><span class="pln"> s </span><span class="kwd">in</span><span class="pln"> slices</span><span class="pun">:</span></code></li><li class="L3"><code class="language-python"><span class="pln">                self</span><span class="pun">.</span><span class="pln">calc_delta</span><span class="pun">(</span><span class="pln">children_delta</span><span class="pun">[</span><span class="pln">s</span><span class="pun">[</span><span class="lit">1</span><span class="pun">]:</span><span class="pln">s</span><span class="pun">[</span><span class="lit">2</span><span class="pun">]],</span><span class="pln"> </span></code></li><li class="L4"><code class="language-python"><span class="pln">                                parent</span><span class="pun">.</span><span class="pln">children</span><span class="pun">[</span><span class="pln">s</span><span class="pun">[</span><span class="lit">0</span><span class="pun">]])</span></code></li><li class="L5"><code class="language-python"></code></li><li class="L6"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> calc_gradient</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> parent</span><span class="pun">):</span></code></li><li class="L7"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L8"><code class="language-python"><span class="str">        计算每个节点权重的梯度，并将它们求和，得到最终的梯度</span></code></li><li class="L9"><code class="language-python"><span class="str">        '''</span></code></li><li class="L0"><code class="language-python"><span class="pln">        W_grad </span><span class="pun">=</span><span class="pln"> np</span><span class="pun">.</span><span class="pln">zeros</span><span class="pun">((</span><span class="pln">self</span><span class="pun">.</span><span class="pln">node_width</span><span class="pun">,</span><span class="pln"> </span></code></li><li class="L1"><code class="language-python"><span class="pln">                            self</span><span class="pun">.</span><span class="pln">node_width </span><span class="pun">*</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">child_count</span><span class="pun">))</span></code></li><li class="L2"><code class="language-python"><span class="pln">        b_grad </span><span class="pun">=</span><span class="pln"> np</span><span class="pun">.</span><span class="pln">zeros</span><span class="pun">((</span><span class="pln">self</span><span class="pun">.</span><span class="pln">node_width</span><span class="pun">,</span><span class="pln"> </span><span class="lit">1</span><span class="pun">))</span></code></li><li class="L3"><code class="language-python"><span class="pln">        </span><span class="kwd">if</span><span class="pln"> </span><span class="kwd">not</span><span class="pln"> parent</span><span class="pun">.</span><span class="pln">children</span><span class="pun">:</span></code></li><li class="L4"><code class="language-python"><span class="pln">            </span><span class="kwd">return</span><span class="pln"> W_grad</span><span class="pun">,</span><span class="pln"> b_grad</span></code></li><li class="L5"><code class="language-python"><span class="pln">        parent</span><span class="pun">.</span><span class="pln">W_grad </span><span class="pun">=</span><span class="pln"> np</span><span class="pun">.</span><span class="pln">dot</span><span class="pun">(</span><span class="pln">parent</span><span class="pun">.</span><span class="pln">delta</span><span class="pun">,</span><span class="pln"> parent</span><span class="pun">.</span><span class="pln">children_data</span><span class="pun">.</span><span class="pln">T</span><span class="pun">)</span></code></li><li class="L6"><code class="language-python"><span class="pln">        parent</span><span class="pun">.</span><span class="pln">b_grad </span><span class="pun">=</span><span class="pln"> parent</span><span class="pun">.</span><span class="pln">delta</span></code></li><li class="L7"><code class="language-python"><span class="pln">        W_grad </span><span class="pun">+=</span><span class="pln"> parent</span><span class="pun">.</span><span class="pln">W_grad</span></code></li><li class="L8"><code class="language-python"><span class="pln">        b_grad </span><span class="pun">+=</span><span class="pln"> parent</span><span class="pun">.</span><span class="pln">b_grad</span></code></li><li class="L9"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> child </span><span class="kwd">in</span><span class="pln"> parent</span><span class="pun">.</span><span class="pln">children</span><span class="pun">:</span></code></li><li class="L0"><code class="language-python"><span class="pln">            W</span><span class="pun">,</span><span class="pln"> b </span><span class="pun">=</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">calc_gradient</span><span class="pun">(</span><span class="pln">child</span><span class="pun">)</span></code></li><li class="L1"><code class="language-python"><span class="pln">            W_grad </span><span class="pun">+=</span><span class="pln"> W</span></code></li><li class="L2"><code class="language-python"><span class="pln">            b_grad </span><span class="pun">+=</span><span class="pln"> b</span></code></li><li class="L3"><code class="language-python"><span class="pln">        </span><span class="kwd">return</span><span class="pln"> W_grad</span><span class="pun">,</span><span class="pln"> b_grad</span></code></li></ol></pre><p data-anchor-id="vjcb">在上述算法中，calc_delta函数和calc_gradient函数分别计算各个节点的误差项<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-94-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -737.2000693679926 451.5 767.4001387359851" style="width: 1.089ex; height: 1.815ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use></g></svg></span><script type="math/tex" id="MathJax-Element-94">\delta</script>以及最终的梯度。它们都采用递归算法，先序遍历整个树，并逐一完成每个节点的计算。</p><p data-anchor-id="i3jr">下面是梯度下降算法的实现（没有weight decay），这个非常简单：</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="602n" style=""><ol class="linenums"><li class="L0"><code class="language-pyhton"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> update</span><span class="pun">(</span><span class="kwd">self</span><span class="pun">):</span></code></li><li class="L1"><code class="language-pyhton"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L2"><code class="language-pyhton"><span class="str">        使用SGD算法更新权重</span></code></li><li class="L3"><code class="language-pyhton"><span class="str">        '''</span></code></li><li class="L4"><code class="language-pyhton"><span class="pln">        </span><span class="kwd">self</span><span class="pun">.</span><span class="pln">W </span><span class="pun">-=</span><span class="pln"> </span><span class="kwd">self</span><span class="pun">.</span><span class="pln">learning_rate </span><span class="pun">*</span><span class="pln"> </span><span class="kwd">self</span><span class="pun">.</span><span class="pln">W_grad</span></code></li><li class="L5"><code class="language-pyhton"><span class="pln">        </span><span class="kwd">self</span><span class="pun">.</span><span class="pln">b </span><span class="pun">-=</span><span class="pln"> </span><span class="kwd">self</span><span class="pun">.</span><span class="pln">learning_rate </span><span class="pun">*</span><span class="pln"> </span><span class="kwd">self</span><span class="pun">.</span><span class="pln">b_grad</span></code></li></ol></pre><p data-anchor-id="icf0">以上就是<strong>递归神经网络</strong>的实现，总共100行左右，和上一篇文章的LSTM相比简单多了。</p><p data-anchor-id="ltz5">最后，我们用梯度检查来验证程序的正确性：</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="ygor" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="kwd">def</span><span class="pln"> gradient_check</span><span class="pun">():</span></code></li><li class="L1"><code class="language-python"><span class="pln">    </span><span class="str">'''</span></code></li><li class="L2"><code class="language-python"><span class="str">    梯度检查</span></code></li><li class="L3"><code class="language-python"><span class="str">    '''</span></code></li><li class="L4"><code class="language-python"><span class="pln">    </span><span class="com"># 设计一个误差函数，取所有节点输出项之和</span></code></li><li class="L5"><code class="language-python"><span class="pln">    error_function </span><span class="pun">=</span><span class="pln"> </span><span class="kwd">lambda</span><span class="pln"> o</span><span class="pun">:</span><span class="pln"> o</span><span class="pun">.</span><span class="pln">sum</span><span class="pun">()</span></code></li><li class="L6"><code class="language-python"></code></li><li class="L7"><code class="language-python"><span class="pln">    rnn </span><span class="pun">=</span><span class="pln"> </span><span class="typ">RecursiveLayer</span><span class="pun">(</span><span class="lit">2</span><span class="pun">,</span><span class="pln"> </span><span class="lit">2</span><span class="pun">,</span><span class="pln"> </span><span class="typ">IdentityActivator</span><span class="pun">(),</span><span class="pln"> </span><span class="lit">1e-3</span><span class="pun">)</span></code></li><li class="L8"><code class="language-python"></code></li><li class="L9"><code class="language-python"><span class="pln">    </span><span class="com"># 计算forward值</span></code></li><li class="L0"><code class="language-python"><span class="pln">    x</span><span class="pun">,</span><span class="pln"> d </span><span class="pun">=</span><span class="pln"> data_set</span><span class="pun">()</span></code></li><li class="L1"><code class="language-python"><span class="pln">    rnn</span><span class="pun">.</span><span class="pln">forward</span><span class="pun">(</span><span class="pln">x</span><span class="pun">[</span><span class="lit">0</span><span class="pun">],</span><span class="pln"> x</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L2"><code class="language-python"><span class="pln">    rnn</span><span class="pun">.</span><span class="pln">forward</span><span class="pun">(</span><span class="pln">rnn</span><span class="pun">.</span><span class="pln">root</span><span class="pun">,</span><span class="pln"> x</span><span class="pun">[</span><span class="lit">2</span><span class="pun">])</span></code></li><li class="L3"><code class="language-python"></code></li><li class="L4"><code class="language-python"><span class="pln">    </span><span class="com"># 求取sensitivity map</span></code></li><li class="L5"><code class="language-python"><span class="pln">    sensitivity_array </span><span class="pun">=</span><span class="pln"> np</span><span class="pun">.</span><span class="pln">ones</span><span class="pun">((</span><span class="pln">rnn</span><span class="pun">.</span><span class="pln">node_width</span><span class="pun">,</span><span class="pln"> </span><span class="lit">1</span><span class="pun">),</span></code></li><li class="L6"><code class="language-python"><span class="pln">                                dtype</span><span class="pun">=</span><span class="pln">np</span><span class="pun">.</span><span class="pln">float64</span><span class="pun">)</span></code></li><li class="L7"><code class="language-python"><span class="pln">    </span><span class="com"># 计算梯度</span></code></li><li class="L8"><code class="language-python"><span class="pln">    rnn</span><span class="pun">.</span><span class="pln">backward</span><span class="pun">(</span><span class="pln">sensitivity_array</span><span class="pun">)</span></code></li><li class="L9"><code class="language-python"></code></li><li class="L0"><code class="language-python"><span class="pln">    </span><span class="com"># 检查梯度</span></code></li><li class="L1"><code class="language-python"><span class="pln">    epsilon </span><span class="pun">=</span><span class="pln"> </span><span class="lit">10e-4</span></code></li><li class="L2"><code class="language-python"><span class="pln">    </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">rnn</span><span class="pun">.</span><span class="pln">W</span><span class="pun">.</span><span class="pln">shape</span><span class="pun">[</span><span class="lit">0</span><span class="pun">]):</span></code></li><li class="L3"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> j </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">rnn</span><span class="pun">.</span><span class="pln">W</span><span class="pun">.</span><span class="pln">shape</span><span class="pun">[</span><span class="lit">1</span><span class="pun">]):</span></code></li><li class="L4"><code class="language-python"><span class="pln">            rnn</span><span class="pun">.</span><span class="pln">W</span><span class="pun">[</span><span class="pln">i</span><span class="pun">,</span><span class="pln">j</span><span class="pun">]</span><span class="pln"> </span><span class="pun">+=</span><span class="pln"> epsilon</span></code></li><li class="L5"><code class="language-python"><span class="pln">            rnn</span><span class="pun">.</span><span class="pln">reset_state</span><span class="pun">()</span></code></li><li class="L6"><code class="language-python"><span class="pln">            rnn</span><span class="pun">.</span><span class="pln">forward</span><span class="pun">(</span><span class="pln">x</span><span class="pun">[</span><span class="lit">0</span><span class="pun">],</span><span class="pln"> x</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L7"><code class="language-python"><span class="pln">            rnn</span><span class="pun">.</span><span class="pln">forward</span><span class="pun">(</span><span class="pln">rnn</span><span class="pun">.</span><span class="pln">root</span><span class="pun">,</span><span class="pln"> x</span><span class="pun">[</span><span class="lit">2</span><span class="pun">])</span></code></li><li class="L8"><code class="language-python"><span class="pln">            err1 </span><span class="pun">=</span><span class="pln"> error_function</span><span class="pun">(</span><span class="pln">rnn</span><span class="pun">.</span><span class="pln">root</span><span class="pun">.</span><span class="pln">data</span><span class="pun">)</span></code></li><li class="L9"><code class="language-python"><span class="pln">            rnn</span><span class="pun">.</span><span class="pln">W</span><span class="pun">[</span><span class="pln">i</span><span class="pun">,</span><span class="pln">j</span><span class="pun">]</span><span class="pln"> </span><span class="pun">-=</span><span class="pln"> </span><span class="lit">2</span><span class="pun">*</span><span class="pln">epsilon</span></code></li><li class="L0"><code class="language-python"><span class="pln">            rnn</span><span class="pun">.</span><span class="pln">reset_state</span><span class="pun">()</span></code></li><li class="L1"><code class="language-python"><span class="pln">            rnn</span><span class="pun">.</span><span class="pln">forward</span><span class="pun">(</span><span class="pln">x</span><span class="pun">[</span><span class="lit">0</span><span class="pun">],</span><span class="pln"> x</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L2"><code class="language-python"><span class="pln">            rnn</span><span class="pun">.</span><span class="pln">forward</span><span class="pun">(</span><span class="pln">rnn</span><span class="pun">.</span><span class="pln">root</span><span class="pun">,</span><span class="pln"> x</span><span class="pun">[</span><span class="lit">2</span><span class="pun">])</span></code></li><li class="L3"><code class="language-python"><span class="pln">            err2 </span><span class="pun">=</span><span class="pln"> error_function</span><span class="pun">(</span><span class="pln">rnn</span><span class="pun">.</span><span class="pln">root</span><span class="pun">.</span><span class="pln">data</span><span class="pun">)</span></code></li><li class="L4"><code class="language-python"><span class="pln">            expect_grad </span><span class="pun">=</span><span class="pln"> </span><span class="pun">(</span><span class="pln">err1 </span><span class="pun">-</span><span class="pln"> err2</span><span class="pun">)</span><span class="pln"> </span><span class="pun">/</span><span class="pln"> </span><span class="pun">(</span><span class="lit">2</span><span class="pln"> </span><span class="pun">*</span><span class="pln"> epsilon</span><span class="pun">)</span></code></li><li class="L5"><code class="language-python"><span class="pln">            rnn</span><span class="pun">.</span><span class="pln">W</span><span class="pun">[</span><span class="pln">i</span><span class="pun">,</span><span class="pln">j</span><span class="pun">]</span><span class="pln"> </span><span class="pun">+=</span><span class="pln"> epsilon</span></code></li><li class="L6"><code class="language-python"><span class="pln">            </span><span class="kwd">print</span><span class="pln"> </span><span class="str">'weights(%d,%d): expected - actural %.4e - %.4e'</span><span class="pln"> </span><span class="pun">%</span><span class="pln"> </span><span class="pun">(</span></code></li><li class="L7"><code class="language-python"><span class="pln">                i</span><span class="pun">,</span><span class="pln"> j</span><span class="pun">,</span><span class="pln"> expect_grad</span><span class="pun">,</span><span class="pln"> rnn</span><span class="pun">.</span><span class="pln">W_grad</span><span class="pun">[</span><span class="pln">i</span><span class="pun">,</span><span class="pln">j</span><span class="pun">])</span></code></li><li class="L8"><code class="language-python"><span class="pln">    </span><span class="kwd">return</span><span class="pln"> rnn</span></code></li></ol></pre><p data-anchor-id="8u6t">下面是梯度检查的结果，完全正确，OH YEAH！</p><p data-anchor-id="d0nc"><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-bea78770357e735a.png" alt="" title=""></p><div class="md-section-divider"></div><h2 data-anchor-id="r14b" id="递归神经网络的应用">递归神经网络的应用</h2><div class="md-section-divider"></div><h3 data-anchor-id="n550" id="自然语言和自然场景解析">自然语言和自然场景解析</h3><p data-anchor-id="2i8a">在自然语言处理任务中，如果我们能够实现一个解析器，将自然语言解析为语法树，那么毫无疑问，这将大大提升我们对自然语言的处理能力。解析器如下所示：</p><p data-anchor-id="mlrz"><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-6b7f075645c12c92.png" alt="" title=""></p><p data-anchor-id="ut9m">可以看出，<strong>递归神经网络</strong>能够完成句子的语法分析，并产生一个语法解析树。</p><p data-anchor-id="sdcu">除了自然语言之外，自然场景也具有<strong>可组合</strong>的性质。因此，我们可以用类似的模型完成自然场景的解析，如下图所示：</p><p data-anchor-id="f86o"><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-7ed5b7d06a8cc583.png" alt="" title=""></p><p data-anchor-id="6az1">两种不同的场景，可以用相同的<strong>递归神经网络</strong>模型来实现。我们以第一个场景，自然语言解析为例。</p><p data-anchor-id="00l2">我们希望将一句话逐字输入到神经网络中，然后，神经网络返回一个解析好的树。为了做到这一点，我们需要给神经网络再加上一层，负责打分。分数越高，说明两个子节点结合更加紧密，分数越低，说明两个子节点结合更松散。如下图所示：</p><p data-anchor-id="l80o"><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-2f0393edbc04c30e.png" alt="" title=""></p><p data-anchor-id="vhe4">一旦这个打分函数训练好了（也就是矩阵U的各项值变为合适的值），我们就可以利用贪心算法来实现句子的解析。第一步，我们先将词按照顺序两两输入神经网络，得到第一组打分：</p><p data-anchor-id="2nyr"><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-be0eb2d1a3527bc6.png" alt="" title=""></p><p data-anchor-id="dbzb">我们发现，现在分数最高的是第一组，The cat，说明它们的结合是最紧密的。这样，我们可以先将它们组合为一个节点。然后，再次两两计算相邻子节点的打分：</p><p data-anchor-id="mh1j"><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-bb8f87bc4f56f5ee.png" alt="" title=""></p><p data-anchor-id="bog5">现在，分数最高的是最后一组，the mat。于是，我们将它们组合为一个节点，再两两计算相邻节点的打分：</p><p data-anchor-id="499a"><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-9c3332923237d11c.png" alt="" title=""></p><p data-anchor-id="yhne">这时，我们发现最高的分数是on the mat，把它们组合为一个节点，继续两两计算相邻节点的打分......最终，我们就能够得到整个解析树：</p><p data-anchor-id="c0k6"><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-14f0140e4eafa5ca.png" alt="" title=""></p><p data-anchor-id="vfla">现在，我们困惑这样牛逼的打分函数score是怎样训练出来的呢？我们需要定义一个目标函数。这里，我们使用Max-Margin目标函数。它的定义如下：</p><div class="md-section-divider"></div><p data-anchor-id="n8gb"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-95-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -970.2000693679926 24254.077881632693 2183.656109255718" style="width: 56.371ex; height: 5.081ex; vertical-align: -2.903ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-4A"></use><use href="#MJMAIN-28" x="633" y="0"></use><use href="#MJMATHI-3B8" x="1023" y="0"></use><use href="#MJMAIN-29" x="1492" y="0"></use><use href="#MJMAIN-3D" x="2159" y="0"></use><use href="#MJMATHI-6D" x="3216" y="0"></use><use href="#MJMATHI-61" x="4094" y="0"></use><use href="#MJMATHI-78" x="4624" y="0"></use><use href="#MJMAIN-28" x="5196" y="0"></use><use href="#MJMAIN-30" x="5586" y="0"></use><use href="#MJMAIN-2C" x="6086" y="0"></use><g transform="translate(6531,0)"><use href="#MJSZ2-2211"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="848" y="-1536"></use></g><g transform="translate(8142,0)"><g transform="translate(286,0)"><use href="#MJMATHI-6D"></use><use href="#MJMATHI-61" x="878" y="0"></use><use href="#MJMATHI-78" x="1408" y="0"></use></g><g transform="translate(0,-710)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2208" x="497" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-41" x="1165" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-28" x="1915" y="0"></use><g transform="translate(1629,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-78"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-69" x="809" y="-213"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="3221" y="0"></use></g></g><use href="#MJMAIN-28" x="10696" y="0"></use><use href="#MJMATHI-73" x="11085" y="0"></use><use href="#MJMAIN-28" x="11555" y="0"></use><g transform="translate(11944,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="809" y="-213"></use></g><use href="#MJMAIN-2C" x="12861" y="0"></use><use href="#MJMATHI-79" x="13306" y="0"></use><use href="#MJMAIN-29" x="13804" y="0"></use><use href="#MJMAIN-2B" x="14416" y="0"></use><use href="#MJMAIN-394" x="15416" y="0"></use><use href="#MJMAIN-28" x="16250" y="0"></use><use href="#MJMATHI-79" x="16639" y="0"></use><use href="#MJMAIN-2C" x="17137" y="0"></use><g transform="translate(17582,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="693" y="-213"></use></g><use href="#MJMAIN-29" x="18417" y="0"></use><use href="#MJMAIN-29" x="18806" y="0"></use><use href="#MJMAIN-2212" x="19418" y="0"></use><use href="#MJMATHI-73" x="20419" y="0"></use><use href="#MJMAIN-28" x="20888" y="0"></use><g transform="translate(21278,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="809" y="-213"></use></g><use href="#MJMAIN-2C" x="22195" y="0"></use><g transform="translate(22640,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="693" y="-213"></use></g><use href="#MJMAIN-29" x="23475" y="0"></use><use href="#MJMAIN-29" x="23864" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-95">
J(\theta)=max(0, \sum_i\underset{y\in A(x_i)}{max}(s(x_i,y)+\Delta(y,y_i))-s(x_i,y_i))
</script><p></p><p data-anchor-id="j6fp">在上式中，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-96-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -462.20006936799257 916.8053928999522 639.8854201102238" style="width: 2.177ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="809" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-96">x_i</script>、<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-97-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 834.8053928999522 688.4001387359851" style="width: 1.935ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="693" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-97">y_i</script>分别表示第i个训练样本的输入和标签，注意这里的标签<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-98-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 834.8053928999522 688.4001387359851" style="width: 1.935ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="693" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-98">y_i</script>是一棵解析树。<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-99-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 3445.2774524665706 1040.4001387359851" style="width: 7.984ex; height: 2.419ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-73"></use><use href="#MJMAIN-28" x="469" y="0"></use><g transform="translate(859,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="809" y="-213"></use></g><use href="#MJMAIN-2C" x="1775" y="0"></use><g transform="translate(2220,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="693" y="-213"></use></g><use href="#MJMAIN-29" x="3055" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-99">s(x_i,y_i)</script>就是打分函数s对第i个训练样本的打分。因为训练样本的标签肯定是正确的，我们希望s对它的打分越高越好，也就是<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-100-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 3445.2774524665706 1040.4001387359851" style="width: 7.984ex; height: 2.419ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-73"></use><use href="#MJMAIN-28" x="469" y="0"></use><g transform="translate(859,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="809" y="-213"></use></g><use href="#MJMAIN-2C" x="1775" y="0"></use><g transform="translate(2220,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="693" y="-213"></use></g><use href="#MJMAIN-29" x="3055" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-100">s(x_i,y_i)</script>越大越好。<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-101-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 2555.9069439838668 1040.4001387359851" style="width: 5.927ex; height: 2.419ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-41"></use><use href="#MJMAIN-28" x="750" y="0"></use><g transform="translate(1140,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="809" y="-213"></use></g><use href="#MJMAIN-29" x="2166" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-101">A(x_1)</script>是所有可能的解析树的集合，而<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-102-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 3107.9720595666186 1040.4001387359851" style="width: 7.258ex; height: 2.419ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-73"></use><use href="#MJMAIN-28" x="469" y="0"></use><g transform="translate(859,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="809" y="-213"></use></g><use href="#MJMAIN-2C" x="1775" y="0"></use><use href="#MJMATHI-79" x="2220" y="0"></use><use href="#MJMAIN-29" x="2718" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-102">s(x_i,y)</script>则是对某个可能的解析树<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-103-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 497.5 688.4001387359851" style="width: 1.21ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use></g></svg></span><script type="math/tex" id="MathJax-Element-103">y</script>的打分。<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-104-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 3389.9720595666186 1040.4001387359851" style="width: 7.863ex; height: 2.419ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-394"></use><use href="#MJMAIN-28" x="833" y="0"></use><use href="#MJMATHI-79" x="1223" y="0"></use><use href="#MJMAIN-2C" x="1720" y="0"></use><g transform="translate(2165,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="693" y="-213"></use></g><use href="#MJMAIN-29" x="3000" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-104">\Delta(y,y_i)</script>是对错误的惩罚。也就是说，如果某个解析树<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-105-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 497.5 688.4001387359851" style="width: 1.21ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use></g></svg></span><script type="math/tex" id="MathJax-Element-105">y</script>和标签<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-106-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 834.8053928999522 688.4001387359851" style="width: 1.935ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="693" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-106">y_i</script>是一样的，那么<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-107-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 3389.9720595666186 1040.4001387359851" style="width: 7.863ex; height: 2.419ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-394"></use><use href="#MJMAIN-28" x="833" y="0"></use><use href="#MJMATHI-79" x="1223" y="0"></use><use href="#MJMAIN-2C" x="1720" y="0"></use><g transform="translate(2165,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="693" y="-213"></use></g><use href="#MJMAIN-29" x="3000" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-107">\Delta(y,y_i)</script>为0，如果网络的输出错的越离谱，那么惩罚项<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-108-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 3389.9720595666186 1040.4001387359851" style="width: 7.863ex; height: 2.419ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-394"></use><use href="#MJMAIN-28" x="833" y="0"></use><use href="#MJMATHI-79" x="1223" y="0"></use><use href="#MJMAIN-2C" x="1720" y="0"></use><g transform="translate(2165,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="693" y="-213"></use></g><use href="#MJMAIN-29" x="3000" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-108">\Delta(y,y_i)</script>的值就越高。<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-109-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 10480.388563577682 1040.4001387359851" style="width: 24.315ex; height: 2.419ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6D"></use><use href="#MJMATHI-61" x="878" y="0"></use><use href="#MJMATHI-78" x="1408" y="0"></use><use href="#MJMAIN-28" x="1980" y="0"></use><use href="#MJMATHI-73" x="2370" y="0"></use><use href="#MJMAIN-28" x="2839" y="0"></use><g transform="translate(3229,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="809" y="-213"></use></g><use href="#MJMAIN-2C" x="4145" y="0"></use><use href="#MJMATHI-79" x="4590" y="0"></use><use href="#MJMAIN-29" x="5088" y="0"></use><use href="#MJMAIN-2B" x="5700" y="0"></use><use href="#MJMAIN-394" x="6700" y="0"></use><use href="#MJMAIN-28" x="7534" y="0"></use><use href="#MJMATHI-79" x="7923" y="0"></use><use href="#MJMAIN-2C" x="8421" y="0"></use><g transform="translate(8866,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="693" y="-213"></use></g><use href="#MJMAIN-29" x="9701" y="0"></use><use href="#MJMAIN-29" x="10090" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-109">max(s(x_i,y)+\Delta(y,y_i))</script>表示所有树里面最高得分。在这里，惩罚项相当于Margin，也就是我们虽然希望打分函数s对正确的树打分比对错误的树打分高，但也不要高过Margin的值。我们优化<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-110-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -725.2000693679926 469.5 755.4001387359851" style="width: 1.089ex; height: 1.815ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B8"></use></g></svg></span><script type="math/tex" id="MathJax-Element-110">\theta</script>，使目标函数取最小值，即：</p><div class="md-section-divider"></div><p data-anchor-id="akic"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-111-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 6971.555555555556 1769.0617008467398" style="width: 16.21ex; height: 4.113ex; vertical-align: -2.419ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B8"></use><use href="#MJMAIN-3D" x="747" y="0"></use><g transform="translate(1803,0)"><use href="#MJMATHI-61"></use><use href="#MJMATHI-72" x="529" y="0"></use><use href="#MJMATHI-67" x="981" y="0"></use><use href="#MJMATHI-6D" x="1461" y="0"></use><use href="#MJMATHI-69" x="2340" y="0"></use><use href="#MJMATHI-6E" x="2685" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-3B8" x="2088" y="-1234"></use></g><use href="#MJMATHI-4A" x="5089" y="0"></use><use href="#MJMAIN-28" x="5723" y="0"></use><use href="#MJMATHI-3B8" x="6112" y="0"></use><use href="#MJMAIN-29" x="6582" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-111">
\theta=\underset{\theta}{argmin}J(\theta)
</script><p></p><p data-anchor-id="7ljf">下面是惩罚函数<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-112-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -736.2000693679926 833.5 756.4001387359851" style="width: 1.935ex; height: 1.815ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-394"></use></g></svg></span><script type="math/tex" id="MathJax-Element-112">\Delta</script>的定义：</p><div class="md-section-divider"></div><p data-anchor-id="7ptq"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-113-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -970.2000693679926 16449.272254573065 2415.5871334849053" style="width: 38.226ex; height: 5.565ex; vertical-align: -3.387ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-394"></use><use href="#MJMAIN-28" x="833" y="0"></use><use href="#MJMATHI-79" x="1223" y="0"></use><use href="#MJMAIN-2C" x="1720" y="0"></use><g transform="translate(2165,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="693" y="-213"></use></g><use href="#MJMAIN-29" x="3000" y="0"></use><use href="#MJMAIN-3D" x="3667" y="0"></use><use href="#MJMATHI-6B" x="4724" y="0"></use><g transform="translate(5412,0)"><use href="#MJSZ2-2211" x="464" y="0"></use><g transform="translate(0,-1149)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-64"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2208" x="523" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-4E" x="1191" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-28" x="2079" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-79" x="2469" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="2966" y="0"></use></g></g><use href="#MJMAINB-31" x="7951" y="0"></use><g transform="translate(8527,0)"><use href="#MJMAIN-7B"></use><use href="#MJMATHI-73" x="500" y="0"></use><use href="#MJMATHI-75" x="970" y="0"></use><use href="#MJMATHI-62" x="1542" y="0"></use><use href="#MJMATHI-54" x="1972" y="0"></use><use href="#MJMATHI-72" x="2676" y="0"></use><use href="#MJMATHI-65" x="3128" y="0"></use><use href="#MJMATHI-65" x="3594" y="0"></use><use href="#MJMAIN-28" x="4061" y="0"></use><use href="#MJMATHI-64" x="4450" y="0"></use><use href="#MJMAIN-29" x="4974" y="0"></use><use href="#MJMAIN-2209" x="5641" y="0"></use><g transform="translate(6586,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="693" y="-213"></use></g><use href="#MJMAIN-7D" x="7421" y="0"></use></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-113">
\Delta(y,y_i)=k\sum_{d\in N(y)}\mathbf{1}{\{subTree(d)\notin y_i\}}
</script><p></p><p data-anchor-id="2hr6">上式中，N(y)是树y节点的集合；subTree(d)是以d为节点的子树。上式的含义是，如果以d为节点的子树没有出现在标签<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-114-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 834.8053928999522 688.4001387359851" style="width: 1.935ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="693" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-114">y_i</script>中，那么函数值+1。最终，惩罚函数的值，是树y中没有出现在树<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-115-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 834.8053928999522 688.4001387359851" style="width: 1.935ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="693" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-115">y_i</script>中的子树的个数，再乘上一个系数k。其实也就是关于两棵树差异的一个度量。</p><p data-anchor-id="jffb"><span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-116-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 2763.666666666667 1040.4001387359851" style="width: 6.411ex; height: 2.419ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-73"></use><use href="#MJMAIN-28" x="469" y="0"></use><use href="#MJMATHI-78" x="859" y="0"></use><use href="#MJMAIN-2C" x="1431" y="0"></use><use href="#MJMATHI-79" x="1876" y="0"></use><use href="#MJMAIN-29" x="2374" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-116">s(x,y)</script>是对一个样本最终的打分，它是对树y每个节点打分的总和。</p><div class="md-section-divider"></div><p data-anchor-id="umwx"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-117-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -970.2000693679926 8857.68002723094 2415.5871334849053" style="width: 20.565ex; height: 5.565ex; vertical-align: -3.387ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-73"></use><use href="#MJMAIN-28" x="469" y="0"></use><use href="#MJMATHI-78" x="859" y="0"></use><use href="#MJMAIN-2C" x="1431" y="0"></use><use href="#MJMATHI-79" x="1876" y="0"></use><use href="#MJMAIN-29" x="2374" y="0"></use><use href="#MJMAIN-3D" x="3041" y="0"></use><g transform="translate(4097,0)"><use href="#MJSZ2-2211" x="1077" y="0"></use><g transform="translate(0,-1149)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2208" x="600" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="1268" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6F" x="1868" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="2354" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-65" x="2877" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-73" x="3343" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-28" x="3813" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-79" x="4203" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="4700" y="0"></use></g></g><g transform="translate(7863,0)"><use href="#MJMATHI-73"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="663" y="-213"></use></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-117">
s(x,y)=\sum_{n\in nodes(y)}s_n
</script><p></p><p data-anchor-id="iy3j">具体细节，读者可以查阅『参考资料3』的论文。</p><div class="md-section-divider"></div><h2 data-anchor-id="rmn3" id="小结">小结</h2><p data-anchor-id="9uda">我们在系列文章中已经介绍的<strong>全连接神经网络</strong>、<strong>卷积神经网络</strong>、<strong>循环神经网络</strong>和<strong>递归神经网络</strong>，在训练时都使用了<strong>监督学习(Supervised Learning)</strong>作为训练方法。在<strong>监督学习</strong>中，每个训练样本既包括输入特征<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-118-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -464.20006936799257 607.5 484.40013873598514" style="width: 1.452ex; height: 1.089ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-78"></use></g></svg></span><script type="math/tex" id="MathJax-Element-118">\mathbf{x}</script>，也包括标记<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-119-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -464.20006936799257 607.5 684.4001387359851" style="width: 1.452ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAINB-79"></use></g></svg></span><script type="math/tex" id="MathJax-Element-119">\mathbf{y}</script>，即样本<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-120-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -913.1292620390899 7205.096948555041 1183.3293314070825" style="width: 16.694ex; height: 2.782ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-64"></use><g transform="translate(524,362)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="734" y="0"></use></g><use href="#MJMAIN-3D" x="1697" y="0"></use><use href="#MJMAIN-7B" x="2753" y="0"></use><g transform="translate(3254,0)"><use href="#MJMAINB-78"></use><g transform="translate(607,362)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="734" y="0"></use></g></g><use href="#MJMAIN-2C" x="4756" y="0"></use><g transform="translate(5201,0)"><use href="#MJMAINB-79"></use><g transform="translate(607,362)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="734" y="0"></use></g></g><use href="#MJMAIN-7D" x="6704" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-120">d^{(i)}=\{\mathbf{x}^{(i)},\mathbf{y}^{(i)}\}</script>。然而，很多情况下，我们无法获得形如<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-121-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -913.1292620390899 4451.449817555213 1183.3293314070825" style="width: 10.282ex; height: 2.782ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-7B"></use><g transform="translate(500,0)"><use href="#MJMAINB-78"></use><g transform="translate(607,362)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="734" y="0"></use></g></g><use href="#MJMAIN-2C" x="2003" y="0"></use><g transform="translate(2448,0)"><use href="#MJMAINB-79"></use><g transform="translate(607,362)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="734" y="0"></use></g></g><use href="#MJMAIN-7D" x="3950" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-121">\{\mathbf{x}^{(i)},\mathbf{y}^{(i)}\}</script>的样本，这时，我们就不能采用<strong>监督学习</strong>的方法。在接下来的几篇文章中，我们重点介绍另外一种学习方法：<strong>增强学习(Reinforcement Learning)</strong>。在了解<strong>增强学习</strong>的主要算法之后，我们还将介绍著名的围棋软件<strong>AlphaGo</strong>，它是一个把<strong>监督学习</strong>和<strong>增强学习</strong>进行完美结合的案例。</p><p data-anchor-id="t3kc"><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-9f3e58723eee0af3.jpg" alt="" title=""></p><div class="md-section-divider"></div><h2 data-anchor-id="me0p" id="参考资料">参考资料</h2><ol data-anchor-id="jlls">
<li><a href="http://cs224d.stanford.edu/" target="_blank">CS224d: Deep Learning for Natural Language Processing</a></li>
<li><a href="https://pdfs.semanticscholar.org/794e/6ed81d21f1bf32a0fd3be05c44c1fa362688.pdf" target="_blank">Learning Task-Dependent Distributed Representations by Back Propagation Through Structure</a></li>
<li><a href="http://ai.stanford.edu/~ang/papers/icml11-ParsingWithRecursiveNeuralNetworks.pdf" target="_blank">Parsing Natural Scenes and Natural Language with Recursive Neural Networks</a></li>
</ol></div>
    <div class="remark-icons"></div>
</div>

<!--in page preview buttons. -->
<div class="in-page-preview-buttons in-page-preview-buttons-full-reader">
    <ul>
        <li class="in-page-button dropdown" id="preview-toc-button" title="内容目录 Ctrl+Alt+O">
            <span class="dropdown-toggle icon-list" data-toggle="dropdown" style="color: rgba(102, 128, 153, 0.45);"></span>
            <div id="toc-list" class="dropdown-menu theme pull-right theme-white"> <!-- Add theme means this element will be changed when apply theme color. -->
                <h3>内容目录</h3>
                <hr>
                <div class="table-of-contents"><div class="toc">
<ul>
<li><a href="https://zybuluo.com/hanbingtao/note/626300#%E9%9B%B6%E5%9F%BA%E7%A1%80%E5%85%A5%E9%97%A8%E6%B7%B1%E5%BA%A6%E5%AD%A6%E4%B9%A07-%E9%80%92%E5%BD%92%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C">零基础入门深度学习(7) - 递归神经网络</a><ul>
<li><a href="https://zybuluo.com/hanbingtao/note/626300#%E6%96%87%E7%AB%A0%E5%88%97%E8%A1%A8">文章列表</a></li>
<li><a href="https://zybuluo.com/hanbingtao/note/626300#%E5%BE%80%E6%9C%9F%E5%9B%9E%E9%A1%BE">往期回顾</a></li>
<li><a href="https://zybuluo.com/hanbingtao/note/626300#%E9%80%92%E5%BD%92%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E6%98%AF%E5%95%A5">递归神经网络是啥</a></li>
<li><a href="https://zybuluo.com/hanbingtao/note/626300#%E9%80%92%E5%BD%92%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%9A%84%E5%89%8D%E5%90%91%E8%AE%A1%E7%AE%97">递归神经网络的前向计算</a></li>
<li><a href="https://zybuluo.com/hanbingtao/note/626300#%E9%80%92%E5%BD%92%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%9A%84%E8%AE%AD%E7%BB%83">递归神经网络的训练</a><ul>
<li><a href="https://zybuluo.com/hanbingtao/note/626300#%E8%AF%AF%E5%B7%AE%E9%A1%B9%E7%9A%84%E4%BC%A0%E9%80%92">误差项的传递</a></li>
<li><a href="https://zybuluo.com/hanbingtao/note/626300#%E6%9D%83%E9%87%8D%E6%A2%AF%E5%BA%A6%E7%9A%84%E8%AE%A1%E7%AE%97">权重梯度的计算</a></li>
<li><a href="https://zybuluo.com/hanbingtao/note/626300#%E6%9D%83%E9%87%8D%E6%9B%B4%E6%96%B0">权重更新</a></li>
</ul>
</li>
<li><a href="https://zybuluo.com/hanbingtao/note/626300#%E9%80%92%E5%BD%92%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%9A%84%E5%AE%9E%E7%8E%B0">递归神经网络的实现</a></li>
<li><a href="https://zybuluo.com/hanbingtao/note/626300#%E9%80%92%E5%BD%92%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%9A%84%E5%BA%94%E7%94%A8">递归神经网络的应用</a><ul>
<li><a href="https://zybuluo.com/hanbingtao/note/626300#%E8%87%AA%E7%84%B6%E8%AF%AD%E8%A8%80%E5%92%8C%E8%87%AA%E7%84%B6%E5%9C%BA%E6%99%AF%E8%A7%A3%E6%9E%90">自然语言和自然场景解析</a></li>
</ul>
</li>
<li><a href="https://zybuluo.com/hanbingtao/note/626300#%E5%B0%8F%E7%BB%93">小结</a></li>
<li><a href="https://zybuluo.com/hanbingtao/note/626300#%E5%8F%82%E8%80%83%E8%B5%84%E6%96%99">参考资料</a></li>
</ul>
</li>
</ul>
</div>
</div>
            </div>
        </li>
    </ul>
</div>

<div id="reader-full-toolbar" class="reader-full-toolbar-shown" style="padding-top: 0;">
    <ul id="reader-full-toolbar-home" class="preview-button-row">
        <li class="preview-button-full-reader" id="preview-editor-button" title="撰写文本 Ctrl+Alt+M">
            <span class="icon-pencil" style="color: rgb(187, 187, 187);"></span>
        </li>
    </ul>
    <ul id="preview-button-row" class="preview-button-row">
        <li class="preview-button-full-reader dropdown" id="preview-list-button" title="文本列表 Ctrl+Alt+F">
            <span class="dropdown-toggle icon-reorder" data-toggle="dropdown" style="color: rgb(187, 187, 187);"></span>
            <ul id="file-list" class="dropdown-menu theme-black pull-right" role="menu" style="max-height: 652px;">
                    <li>
                    <ul class="tag-list">
                        <li class="tag-item item" tag-name="机器学习">
                            <span class="pull-left"><i class="icon-tag"></i><span class="tag-name">机器学习</span></span>
                            <span class="tag-count pull-right">7</span>
                            <div class="clearfix"></div>
                        </li>
                            
    <li class="file-item item" file-created-date="2017-02-28 00:59:57">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/626300" title="【已发布】 2017-08-29 23:41">
        <i class="icon-share-sign"></i>
        <span id="626300" class="whiter-on-black">零基础入门深度学习(7) - 递归神经网络</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2017-01-08 23:28:36">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/581764" title="【已发布】 2017-08-28 19:55">
        <i class="icon-share-sign"></i>
        <span id="581764">零基础入门深度学习(6) - 长短时记忆网络(LSTM)</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-11-05 19:16:51">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/541458" title="【已发布】 2017-08-28 19:54">
        <i class="icon-share-sign"></i>
        <span id="541458">零基础入门深度学习(5) - 循环神经网络</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-10-09 20:30:46">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/485480" title="【已发布】 2017-08-28 19:53">
        <i class="icon-share-sign"></i>
        <span id="485480">零基础入门深度学习(4) - 卷积神经网络</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-08-24 21:39:25">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/476663" title="【已发布】 2017-10-17 22:25">
        <i class="icon-share-sign"></i>
        <span id="476663">零基础入门深度学习(3) - 神经网络和反向传播算法</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-07-26 01:44:30">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/448086" title="【已发布】 2017-08-28 19:40">
        <i class="icon-share-sign"></i>
        <span id="448086">零基础入门深度学习(2) - 线性单元和梯度下降</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-07-12 11:10:42">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/433855" title="【已发布】 2017-08-28 19:35">
        <i class="icon-share-sign"></i>
        <span id="433855">零基础入门深度学习(1) - 感知器</span>
        </a>
    </li>

                    </ul>
                    </li>
                    <li>
                    <ul class="tag-list">
                        <li class="tag-item item" tag-name="深度学习入门">
                            <span class="pull-left"><i class="icon-tag"></i><span class="tag-name">深度学习入门</span></span>
                            <span class="tag-count pull-right">7</span>
                            <div class="clearfix"></div>
                        </li>
                            
    <li class="file-item item" file-created-date="2017-02-28 00:59:57">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/626300" title="【已发布】 2017-08-29 23:41">
        <i class="icon-share-sign"></i>
        <span id="626300" class="whiter-on-black">零基础入门深度学习(7) - 递归神经网络</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2017-01-08 23:28:36">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/581764" title="【已发布】 2017-08-28 19:55">
        <i class="icon-share-sign"></i>
        <span id="581764">零基础入门深度学习(6) - 长短时记忆网络(LSTM)</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-11-05 19:16:51">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/541458" title="【已发布】 2017-08-28 19:54">
        <i class="icon-share-sign"></i>
        <span id="541458">零基础入门深度学习(5) - 循环神经网络</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-10-09 20:30:46">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/485480" title="【已发布】 2017-08-28 19:53">
        <i class="icon-share-sign"></i>
        <span id="485480">零基础入门深度学习(4) - 卷积神经网络</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-08-24 21:39:25">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/476663" title="【已发布】 2017-10-17 22:25">
        <i class="icon-share-sign"></i>
        <span id="476663">零基础入门深度学习(3) - 神经网络和反向传播算法</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-07-26 01:44:30">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/448086" title="【已发布】 2017-08-28 19:40">
        <i class="icon-share-sign"></i>
        <span id="448086">零基础入门深度学习(2) - 线性单元和梯度下降</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-07-12 11:10:42">
        <a tabindex="-1" href="https://zybuluo.com/hanbingtao/note/433855" title="【已发布】 2017-08-28 19:35">
        <i class="icon-share-sign"></i>
        <span id="433855">零基础入门深度学习(1) - 感知器</span>
        </a>
    </li>

                    </ul>
                    </li>
            </ul>
            <ul id="file-list-topbar" class="dropdown-menu theme-black pull-right" role="menu">
                <li id="search-file-bar">
                    <i class="icon-search icon-large"></i>
                    <input type="text" id="search-file-textbox" placeholder="搜索 hanbingtao 的文稿标题， * 显示全部">
                    <i class="icon-level-down icon-rotate-90 icon-large"></i>
                </li>
                <li id="tag-file-bar">
                    以下【标签】将用于标记这篇文稿：
                </li>
            </ul>
        </li>
        <li class="preview-button-full-reader" id="preview-theme-button" title="主题切换 Ctrl+Alt+Y">
            <span class="icon-adjust" style="color: rgb(187, 187, 187);"></span>
        </li>
        <li class="preview-button-full-reader" id="preview-fullscreen-button" title="全屏模式 F11">
            <span class="icon-fullscreen" style="color: rgb(187, 187, 187);"></span>
        </li>
        <li class="preview-button-full-reader wmd-spacer"></li>
        <li class="preview-button-full-reader dropdown" id="preview-about-button" title="关于本站">
            <span class="dropdown-toggle icon-info-sign" data-toggle="dropdown" data-hover="dropdown" data-delay="100" data-close-others="true" style="color: rgb(187, 187, 187);"></span>
            <ul id="about-menu" class="dropdown-menu theme-black pull-right" role="menu">
                <li title="下载全平台客户端"><a tabindex="-1" href="https://www.zybuluo.com/cmd" target="_blank"><i class="icon-laptop"></i>下载客户端</a></li>
                <li title="@ghosert"><a tabindex="-1" href="http://www.weibo.com/ghosert" target="_blank"><i class="icon-weibo"></i>关注开发者</a></li>
                <li title=""><a tabindex="-1" href="https://github.com/ghosert/cmd-editor/issues" target="_blank"><i class="icon-github-alt"></i>报告问题，建议</a></li>
                <li title="support@zybuluo.com"><a tabindex="-1" href="mailto:support@zybuluo.com" target="_blank"><i class="icon-envelope"></i>联系我们</a></li>
            </ul>
        </li>
    </ul>
</div>
<ul id="reader-full-toolbar-tail" class="reader-full-toolbar-tail-shown">
    <li class="preview-button-full-reader" id="preview-hidden-button" title="隐藏工具栏 Ctrl+Alt+I">
        <span class="icon-chevron-sign-right" style="color: rgb(187, 187, 187);"></span>
    </li>
</ul>






<!-- side remark, hidden when loading. -->
<div class="remark-list side-remark-hidden">
    <div class="remark-items">
    </div>
    <div class="leave-remark unselectable"><span class="icon-plus-sign-alt"></span><span>添加新批注</span></div>
    <div class="new-remark">
    <div class="remark-head"><a><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/default-head.jpg" style="width: 32px; height: 32px;"></a></div>
    <div class="remark-author unselectable"><strong></strong></div>
    <div class="remark-editor" contenteditable="true" spellcheck="false"></div>
    <!-- this will be filled up by js.
    <div class="inline-error">402/400</div> for new remark
    <div class="inline-error">202/200</div> for new reply
    -->
    <div class="remark-footer unselectable">
        <button class="remark-save btn-link">保存</button>
        <button class="remark-cancel btn-link">取消</button>
    </div>

        <!-- clone the template $('.new-remark-reply').html() to here.-->
        <div class="remark-notice">在作者公开此批注前，只有你和作者可见。</div>
    </div>
</div>

<!-- template for new remark/reply -->
<div class="new-remark-reply side-remark-hidden">
    <div class="remark-head"><a><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/default-head.jpg"></a></div>
    <div class="remark-author unselectable"></div>
    <div class="remark-editor" contenteditable="true" spellcheck="false"></div>
    <!-- this will be filled up by js.
    <div class="inline-error">402/400</div> for new remark
    <div class="inline-error">202/200</div> for new reply
    -->
    <div class="remark-footer unselectable">
        <button class="remark-save btn-link">保存</button>
        <button class="remark-cancel btn-link">取消</button>
    </div>
</div>

<!-- template for .remark-item/.remark-reply -->
<div class="remark-item-reply side-remark-hidden">
    <div class="remark-head"><a><img src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/default-head.jpg"></a></div>
    <div class="remark-author unselectable"></div>
    <div class="remark-delete-link unselectable"><span class="icon-remove"></span></div> <!--This is mainly for deleting remark-reply, shown when author/remark hovering on remark-reply.-->
    <div class="remark-editor" contenteditable="true" spellcheck="false"></div>
    <!-- this will be filled up by js.
    <div class="inline-error">402/400</div> for new remark
    <div class="inline-error">202/200</div> for new reply
    -->
    <div class="remark-footer unselectable">
        <button class="remark-edit btn-link">修改</button>
        <button class="remark-save btn-link">保存</button>
        <button class="remark-cancel btn-link">取消</button>
        <button class="remark-delete btn-link">删除</button>
    </div>
</div>

<!-- template for remark-item-->
<div class="remark-item side-remark-hidden" data-rand-id="" data-version-id="">
    <div class="remark-published-link unselectable"><span class="icon-link icon-rotate-90"></span></div>
    <ul class="remark-options theme unselectable theme-white">
        <li class="remark-private"><span class="icon-eye-close"></span><span>私有</span></li>
        <li class="remark-public"><span class="icon-group"></span><span>公开</span></li>
        <li class="remark-delete"><span class="icon-remove"></span><span>删除</span></li>
    </ul>

    <!-- clone the template $('.remark-item-reply').html() to here.-->

    <button class="remark-reply-view-more btn-link">查看更早的 5 条回复</button>
    <div class="remark-replies">
        <!--
        <div class="remark-reply">
            clone the template $('.remark-item-reply').html() to here.
        </div>
        -->
    </div>

    <div class="leave-reply unselectable"><span>回复批注</span></div>
    <div class="new-reply">
        <!-- clone the template $('.new-remark-reply').html() to here.-->
    </div>
</div>

<!-- jiawzhang NOTICE: .remark-icons will be put to mdeditor.mako and user_note.mako, where next to .wmd-preview -->
<!-- <div class="remark-icons"></div> -->

<!-- template for remark-icon -->
<div class="remark-icon unselectable side-remark-hidden remark-icon-empty" style="display: none;">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div>


<!-- canvas, hidden always, this is used to convert svg to canvas and then convert canvas to png. -->
<canvas id="svg-canvas-image" class="editor-reader-hidden-always"></canvas>

<!-- This is the image panel to hold enlarged image/svg. -->
<div id="large-image-panel">
    <img id="large-image">
</div>


    


    <!-- Hidden Popup Modal -->
    <div id="notification-popup-window" class="modal hide fade theme theme-white" tabindex="-1" role="dialog" aria-labelledby="notification-title" aria-hidden="true">
        <div class="modal-header theme theme-white">
            <button type="button" class="close" data-dismiss="modal" aria-hidden="true">×</button>
            <h3 id="notification-title">通知</h3>
        </div>
        <div class="modal-body theme theme-white">
            <p></p>
        </div>
        <div class="modal-footer theme theme-white">
            <button id="notification-cancel" class="btn" data-dismiss="modal" aria-hidden="true">取消</button>
            <button id="notification-confirm" class="btn btn-primary">确认</button>
        </div>
    </div>

    <!-- zybuluo's foot -->

    <script src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/288313bb.base.lib.min.js"></script>

    <script>
        Namespace('com.zybuluo.base');
        com.zybuluo.base.initData = {
            globalPromptUrl: "https://zybuluo.com/global/prompt",
        };
    </script>

    
    <!--mathjax-->
    <!--blacker: 1 below means font weight.-->
    <script type="text/x-mathjax-config;executed=true">
        MathJax.Hub.Config({ tex2jax: { inlineMath: [['$','$'], ["\\(","\\)"]], processEscapes: true }, TeX: { equationNumbers: { autoNumber: "AMS" } }, messageStyle: "none", SVG: { blacker: 1 }});
    </script>
    <script src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/mathJax.js"></script>
    <!--mathjax source code is here: https://github.com/mathjax/MathJax.-->
    <script src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/mathJax(1).js"></script>

    <script>
        Namespace('com.zybuluo.mdeditor.layout');
        com.zybuluo.mdeditor.layout.initData = {
            // '' means not logged in, otherwise the logged in username, for mdeditor.mako, this value will be reset in render.js otherwise, for user_note.mako, it's rendered by server side.
            loggedInUsername: '',
            isPageOwner: 'False' === 'True' ? true : false,
            loginComeFromUrl: 'https://zybuluo.com/login?return_to=https%3A%2F%2Fzybuluo.com%2Fhanbingtao%2Fnote%2F626300',
            noteRemarksUrl: "https://zybuluo.com/note/626300/remarks", 
            newNoteRemarkUrl: "https://zybuluo.com/note/626300/remark/new", 
            updateNoteRemarkUrl: "https://zybuluo.com/note/626300/remark/update", 
            deleteNoteRemarkUrl: "https://zybuluo.com/note/626300/remark/delete", 
            publishNoteRemarkUrl: "https://zybuluo.com/note/626300/remark/publish", 
            newNoteRemarkReplyUrl: "https://zybuluo.com/note/626300/remark_reply/new", 
            updateNoteRemarkReplyUrl: "https://zybuluo.com/note/626300/remark_reply/update", 
            deleteNoteRemarkReplyUrl: "https://zybuluo.com/note/626300/remark_reply/delete", 
        };

        // BEGIN: pace.js configuration
        window.paceOptions = {
            // disable others, enable for ajax call only,
            ajax: true,
            document: false,
            elements: false,
            eventLag: false,
        };
        // jiawzhang NOTICE: to make sure pace.js is working for any ajax call especially the jquery ajax, add 'Pace.restart()' into jquery ajax call like '$.post'
        // Originally, pace 0.5.6 doesn't support jquery ajax, see details in: https://github.com/HubSpot/pace/issues/29
        // END: pace.js configuration

    </script>

    <script src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/7a70106e.layout.lib.min.js"></script>

    <script src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/dc648f35.layout.min.js"></script><div id="medium-editor-toolbar-1" class="medium-editor-toolbar"><ul id="medium-editor-toolbar-actions" class="medium-editor-toolbar-actions clearfix"><li><button data-action="remark" class=" medium-editor-button-first"><i class="icon-comment"></i></button></li><li><button data-action="highlight" class=" medium-editor-button-last"><i class="icon-pencil"></i></button></li></ul></div>



    

    <!-- https://zybuluo.com/static/assets/mdeditor/user_note.lib.min.js -->
    <!-- -->

    <script>
        Namespace('com.zybuluo.mdeditor.user_note');
        com.zybuluo.mdeditor.user_note.initData = {
            isLoggedIn: 'False',
            mdeditorUrl: "https://zybuluo.com/mdeditor",
            passwordPassed: 'True' === 'True' ? true : false,
        };
    </script>

    <script src="./零基础入门深度学习(7) - 递归神经网络 - 作业部落 Cmd Markdown 编辑阅读器_files/6cd3112e.user_note.min.js"></script>





    
</body></html>